diff --git a/.circleci/config.yml b/.circleci/config.yml
index 90afb1ce29684..ea93575ac9430 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -3,7 +3,7 @@ version: 2.1
jobs:
test-arm:
machine:
- image: ubuntu-2004:2022.04.1
+ image: default
resource_class: arm.large
environment:
ENV_FILE: ci/deps/circle-310-arm64.yaml
@@ -46,7 +46,7 @@ jobs:
cibw-build:
type: string
machine:
- image: ubuntu-2004:2022.04.1
+ image: default
resource_class: arm.large
environment:
TRIGGER_SOURCE: << pipeline.trigger_source >>
diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml
index b4778b74df335..66e4142dc0cbb 100644
--- a/.github/actions/run-tests/action.yml
+++ b/.github/actions/run-tests/action.yml
@@ -1,16 +1,9 @@
name: Run tests and report results
-inputs:
- preload:
- description: Preload arguments for sanitizer
- required: false
- asan_options:
- description: Arguments for Address Sanitizer (ASAN)
- required: false
runs:
using: composite
steps:
- name: Test
- run: ${{ inputs.asan_options }} ${{ inputs.preload }} ci/run_tests.sh
+ run: ci/run_tests.sh
shell: bash -el {0}
- name: Publish test results
@@ -20,11 +13,6 @@ runs:
path: test-data.xml
if: failure()
- - name: Report Coverage
- run: coverage report -m
- shell: bash -el {0}
- if: failure()
-
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
index 8e29d56f47dcf..24b2de251ce8e 100644
--- a/.github/workflows/code-checks.yml
+++ b/.github/workflows/code-checks.yml
@@ -86,7 +86,7 @@ jobs:
if: ${{ steps.build.outcome == 'success' && always() }}
- name: Typing + pylint
- uses: pre-commit/action@v3.0.0
+ uses: pre-commit/action@v3.0.1
with:
extra_args: --verbose --hook-stage manual --all-files
if: ${{ steps.build.outcome == 'success' && always() }}
diff --git a/.github/workflows/comment-commands.yml b/.github/workflows/comment-commands.yml
index 425abef850184..62956f5825782 100644
--- a/.github/workflows/comment-commands.yml
+++ b/.github/workflows/comment-commands.yml
@@ -24,12 +24,10 @@ jobs:
concurrency:
group: ${{ github.actor }}-preview-docs
steps:
- - run: |
- if curl --output /dev/null --silent --head --fail "https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"; then
- curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "Website preview of this PR available at: https://pandas.pydata.org/preview/${{ github.event.issue.number }}/"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
- else
- curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"body": "No preview found for PR #${{ github.event.issue.number }}. Did the docs build complete?"}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments
- fi
+ - uses: pandas-dev/github-doc-previewer@v0.3.2
+ with:
+ previewer-server: "https://pandas.pydata.org/preview"
+ artifact-job: "Doc Build and Upload"
asv_run:
runs-on: ubuntu-22.04
# TODO: Support more benchmarking options later, against different branches, against self, etc
diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml
index 73acd9acc129a..924a6263f34d2 100644
--- a/.github/workflows/docbuild-and-upload.yml
+++ b/.github/workflows/docbuild-and-upload.yml
@@ -90,10 +90,3 @@ jobs:
name: website
path: web/build
retention-days: 14
-
- - name: Trigger web/doc preview
- run: curl -X POST https://pandas.pydata.org/preview/submit/$RUN_ID/$PR_ID/
- env:
- RUN_ID: ${{ github.run_id }}
- PR_ID: ${{ github.event.pull_request.number }}
- if: github.event_name == 'pull_request'
diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml
index d59ddf272f705..2de1649d42dfd 100644
--- a/.github/workflows/package-checks.yml
+++ b/.github/workflows/package-checks.yml
@@ -24,7 +24,7 @@ jobs:
runs-on: ubuntu-22.04
strategy:
matrix:
- extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "consortium-standard", "all"]
+ extra: ["test", "pyarrow", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output-formatting", "clipboard", "compression", "all"]
fail-fast: false
name: Install Extras - ${{ matrix.extra }}
concurrency:
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index dd5d090e098b0..f93950224eaae 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -57,34 +57,6 @@ jobs:
# Also install zh_CN (its encoding is gb2312) but do not activate it.
# It will be temporarily activated during tests with locale.setlocale
extra_loc: "zh_CN"
- - name: "Copy-on-Write 3.9"
- env_file: actions-39.yaml
- pattern: "not slow and not network and not single_cpu"
- pandas_copy_on_write: "1"
- - name: "Copy-on-Write 3.10"
- env_file: actions-310.yaml
- pattern: "not slow and not network and not single_cpu"
- pandas_copy_on_write: "1"
- - name: "Copy-on-Write 3.11"
- env_file: actions-311.yaml
- pattern: "not slow and not network and not single_cpu"
- pandas_copy_on_write: "1"
- - name: "Copy-on-Write 3.12"
- env_file: actions-312.yaml
- pattern: "not slow and not network and not single_cpu"
- pandas_copy_on_write: "1"
- - name: "Copy-on-Write 3.11 (warnings)"
- env_file: actions-311.yaml
- pattern: "not slow and not network and not single_cpu"
- pandas_copy_on_write: "warn"
- - name: "Copy-on-Write 3.10 (warnings)"
- env_file: actions-310.yaml
- pattern: "not slow and not network and not single_cpu"
- pandas_copy_on_write: "warn"
- - name: "Copy-on-Write 3.9 (warnings)"
- env_file: actions-39.yaml
- pattern: "not slow and not network and not single_cpu"
- pandas_copy_on_write: "warn"
- name: "Pypy"
env_file: actions-pypy-39.yaml
pattern: "not slow and not network and not single_cpu"
@@ -96,21 +68,12 @@ jobs:
- name: "Pyarrow Nightly"
env_file: actions-311-pyarrownightly.yaml
pattern: "not slow and not network and not single_cpu"
- - name: "ASAN / UBSAN"
- env_file: actions-311-sanitizers.yaml
- pattern: "not slow and not network and not single_cpu and not skip_ubsan"
- asan_options: "ASAN_OPTIONS=detect_leaks=0"
- preload: LD_PRELOAD=$(gcc -print-file-name=libasan.so)
- meson_args: --config-settings=setup-args="-Db_sanitize=address,undefined"
- cflags_adds: -fno-sanitize-recover=all
- pytest_workers: -1 # disable pytest-xdist as it swallows stderr from ASAN
fail-fast: false
name: ${{ matrix.name || format('ubuntu-latest {0}', matrix.env_file) }}
env:
PATTERN: ${{ matrix.pattern }}
LANG: ${{ matrix.lang || 'C.UTF-8' }}
LC_ALL: ${{ matrix.lc_all || '' }}
- PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
TEST_ARGS: ${{ matrix.test_args || '' }}
PYTEST_WORKERS: ${{ matrix.pytest_workers || 'auto' }}
@@ -120,7 +83,7 @@ jobs:
QT_QPA_PLATFORM: offscreen
concurrency:
# https://github.community/t/concurrecy-not-work-for-push/183068/7
- group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_copy_on_write || '' }}
+ group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}}
cancel-in-progress: true
services:
@@ -190,18 +153,12 @@ jobs:
- name: Test (not single_cpu)
uses: ./.github/actions/run-tests
if: ${{ matrix.name != 'Pypy' }}
- with:
- preload: ${{ matrix.preload }}
- asan_options: ${{ matrix.asan_options }}
env:
# Set pattern to not single_cpu if not already set
PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}
- name: Test (single_cpu)
uses: ./.github/actions/run-tests
- with:
- preload: ${{ matrix.preload }}
- asan_options: ${{ matrix.asan_options }}
env:
PATTERN: 'single_cpu'
PYTEST_WORKERS: 0
@@ -211,7 +168,8 @@ jobs:
timeout-minutes: 90
strategy:
matrix:
- os: [macos-latest, windows-latest]
+ # Note: Don't use macOS latest since macos 14 appears to be arm64 only
+ os: [macos-13, macos-14, windows-latest]
env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml, actions-312.yaml]
fail-fast: false
runs-on: ${{ matrix.os }}
@@ -224,8 +182,7 @@ jobs:
PANDAS_CI: 1
PYTEST_TARGET: pandas
PATTERN: "not slow and not db and not network and not single_cpu"
- # GH 47443: PYTEST_WORKERS > 0 crashes Windows builds with memory related errors
- PYTEST_WORKERS: ${{ matrix.os == 'macos-latest' && 'auto' || '0' }}
+ PYTEST_WORKERS: 'auto'
steps:
- name: Checkout
@@ -351,7 +308,8 @@ jobs:
strategy:
fail-fast: false
matrix:
- os: [ubuntu-22.04, macOS-latest, windows-latest]
+ # Separate out macOS 13 and 14, since macOS 14 is arm64 only
+ os: [ubuntu-22.04, macOS-13, macOS-14, windows-latest]
timeout-minutes: 90
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
index 841559c8e9799..470c044d2e99e 100644
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@@ -94,7 +94,9 @@ jobs:
buildplat:
- [ubuntu-22.04, manylinux_x86_64]
- [ubuntu-22.04, musllinux_x86_64]
- - [macos-12, macosx_*]
+ - [macos-12, macosx_x86_64]
+ # Note: M1 images on Github Actions start from macOS 14
+ - [macos-14, macosx_arm64]
- [windows-2022, win_amd64]
# TODO: support PyPy?
python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
@@ -128,7 +130,7 @@ jobs:
# Python version used to build sdist doesn't matter
# wheel will be built from sdist with the correct version
- name: Unzip sdist (macOS)
- if: ${{ matrix.buildplat[1] == 'macosx_*' }}
+ if: ${{ startsWith(matrix.buildplat[1], 'macosx') }}
run: |
tar -xzf ./dist/${{ env.sdist_name }} -C ./dist
@@ -139,18 +141,18 @@ jobs:
- name: Build normal wheels
if: ${{ (env.IS_SCHEDULE_DISPATCH != 'true' || env.IS_PUSH == 'true') }}
- uses: pypa/cibuildwheel@v2.16.2
+ uses: pypa/cibuildwheel@v2.17.0
with:
- package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
+ package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
env:
CIBW_PRERELEASE_PYTHONS: True
CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
- name: Build nightly wheels (with NumPy pre-release)
if: ${{ (env.IS_SCHEDULE_DISPATCH == 'true' && env.IS_PUSH != 'true') }}
- uses: pypa/cibuildwheel@v2.16.2
+ uses: pypa/cibuildwheel@v2.17.0
with:
- package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
+ package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
env:
# The nightly wheels should be build witht he NumPy 2.0 pre-releases
# which requires the additional URL.
@@ -183,7 +185,7 @@ jobs:
$TST_CMD = @"
python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0;
python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
- python -c `'import pandas as pd; pd.test(extra_args=[\"`\"--no-strict-data-files`\"\", \"`\"-m not clipboard and not single_cpu and not slow and not network and not db`\"\"])`';
+ python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`';
"@
# add rc to the end of the image name if the Python version is unreleased
docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }}
@@ -191,7 +193,7 @@ jobs:
- uses: actions/upload-artifact@v4
with:
- name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
+ name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
path: ./wheelhouse/*.whl
- name: Upload wheels & sdist
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4b02ad7cf886f..41f1c4c6892a3 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,13 +18,8 @@ ci:
# manual stage hooks
skip: [pylint, pyright, mypy]
repos:
-- repo: https://github.com/hauntsaninja/black-pre-commit-mirror
- # black compiled with mypyc
- rev: 23.11.0
- hooks:
- - id: black
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.1.6
+ rev: v0.3.1
hooks:
- id: ruff
args: [--exit-non-zero-on-fix]
@@ -35,6 +30,14 @@ repos:
files: ^pandas
exclude: ^pandas/tests
args: [--select, "ANN001,ANN2", --fix-only, --exit-non-zero-on-fix]
+ - id: ruff
+ name: ruff-use-pd_array-in-core
+ alias: ruff-use-pd_array-in-core
+ files: ^pandas/core/
+ exclude: ^pandas/core/api\.py$
+ args: [--select, "ICN001", --exit-non-zero-on-fix]
+ - id: ruff-format
+ exclude: ^scripts
- repo: https://github.com/jendrikseipp/vulture
rev: 'v2.10'
hooks:
@@ -56,13 +59,11 @@ repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:
- - id: check-ast
- id: check-case-conflict
- id: check-toml
- id: check-xml
- id: check-yaml
exclude: ^ci/meta.yaml$
- - id: debug-statements
- id: end-of-file-fixer
exclude: \.txt$
- id: mixed-line-ending
@@ -77,7 +78,7 @@ repos:
hooks:
- id: pylint
stages: [manual]
- args: [--load-plugins=pylint.extensions.redefined_loop_name]
+ args: [--load-plugins=pylint.extensions.redefined_loop_name, --fail-on=I0021]
- id: pylint
alias: redefined-outer-name
name: Redefining name from outer scope
@@ -103,8 +104,6 @@ repos:
- repo: https://github.com/pre-commit/pygrep-hooks
rev: v1.10.0
hooks:
- - id: python-check-blanket-noqa
- - id: python-check-blanket-type-ignore
- id: rst-backticks
- id: rst-directive-colons
types: [text] # overwrite types: [rst]
@@ -134,7 +133,7 @@ repos:
types: [python]
stages: [manual]
additional_dependencies: &pyright_dependencies
- - pyright@1.1.339
+ - pyright@1.1.351
- id: pyright
# note: assumes python env is setup and activated
name: pyright reportGeneralTypeIssues
@@ -192,9 +191,6 @@ repos:
# Check for deprecated messages without sphinx directive
|(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)
- # {foo!r} instead of {repr(foo)}
- |!r}
-
# builtin filter function
|(?
-
+
+
-----------------
diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json
index 9b0dc14fe6747..e02ff26ba14e9 100644
--- a/asv_bench/asv.conf.json
+++ b/asv_bench/asv.conf.json
@@ -41,7 +41,7 @@
// pip (with all the conda available packages installed first,
// followed by the pip installed packages).
"matrix": {
- "Cython": ["3.0.5"],
+ "Cython": ["3.0"],
"matplotlib": [],
"sqlalchemy": [],
"scipy": [],
diff --git a/asv_bench/benchmarks/algos/isin.py b/asv_bench/benchmarks/algos/isin.py
index 2b3d32fb579dc..a17732c70c2c7 100644
--- a/asv_bench/benchmarks/algos/isin.py
+++ b/asv_bench/benchmarks/algos/isin.py
@@ -59,10 +59,11 @@ def setup(self, dtype):
elif dtype in ["str", "string[python]", "string[pyarrow]"]:
try:
self.series = Series(
- Index([f"i-{i}" for i in range(N)], dtype=object), dtype=dtype
+ Index([f"i-{i}" for i in range(N)], dtype=object)._values,
+ dtype=dtype,
)
- except ImportError:
- raise NotImplementedError
+ except ImportError as err:
+ raise NotImplementedError from err
self.values = list(self.series[:2])
else:
diff --git a/asv_bench/benchmarks/array.py b/asv_bench/benchmarks/array.py
index aefc6e6d3c307..953af5c868356 100644
--- a/asv_bench/benchmarks/array.py
+++ b/asv_bench/benchmarks/array.py
@@ -76,8 +76,8 @@ class ArrowStringArray:
def setup(self, multiple_chunks):
try:
import pyarrow as pa
- except ImportError:
- raise NotImplementedError
+ except ImportError as err:
+ raise NotImplementedError from err
strings = np.array([str(i) for i in range(10_000)], dtype=object)
if multiple_chunks:
chunks = [strings[i : i + 100] for i in range(0, len(strings), 100)]
diff --git a/asv_bench/benchmarks/categoricals.py b/asv_bench/benchmarks/categoricals.py
index 1716110b619d6..69697906e493e 100644
--- a/asv_bench/benchmarks/categoricals.py
+++ b/asv_bench/benchmarks/categoricals.py
@@ -88,7 +88,7 @@ def setup(self):
)
for col in ("int", "float", "timestamp"):
- self.df[col + "_as_str"] = self.df[col].astype(str)
+ self.df[f"{col}_as_str"] = self.df[col].astype(str)
for col in self.df.columns:
self.df[col] = self.df[col].astype("category")
diff --git a/asv_bench/benchmarks/frame_methods.py b/asv_bench/benchmarks/frame_methods.py
index a283afd1f0f1e..ce31d63f0b70f 100644
--- a/asv_bench/benchmarks/frame_methods.py
+++ b/asv_bench/benchmarks/frame_methods.py
@@ -159,12 +159,6 @@ def setup(self):
def time_items(self):
# (monitor no-copying behaviour)
- if hasattr(self.df, "_item_cache"):
- self.df._item_cache.clear()
- for name, col in self.df.items():
- pass
-
- def time_items_cached(self):
for name, col in self.df.items():
pass
@@ -593,8 +587,6 @@ def setup(self):
N = 10000
# this is the worst case, where every column has NaNs.
arr = np.random.randn(N, 100)
- # NB: we need to set values in array, not in df.values, otherwise
- # the benchmark will be misleading for ArrayManager
arr[::2] = np.nan
self.df = DataFrame(arr)
diff --git a/asv_bench/benchmarks/index_object.py b/asv_bench/benchmarks/index_object.py
index 637b1b40f59a3..9c1e9656503f7 100644
--- a/asv_bench/benchmarks/index_object.py
+++ b/asv_bench/benchmarks/index_object.py
@@ -136,10 +136,14 @@ def setup(self):
self.int_idxs.append(i_idx)
o_idx = i_idx.astype(str)
self.object_idxs.append(o_idx)
+ self.same_range_idx = [self.range_idx] * N
def time_append_range_list(self):
self.range_idx.append(self.range_idxs)
+ def time_append_range_list_same(self):
+ self.range_idx.append(self.same_range_idx)
+
def time_append_int_list(self):
self.int_idx.append(self.int_idxs)
diff --git a/asv_bench/benchmarks/indexing.py b/asv_bench/benchmarks/indexing.py
index 9ad1f5b31016d..15e691d46f693 100644
--- a/asv_bench/benchmarks/indexing.py
+++ b/asv_bench/benchmarks/indexing.py
@@ -3,6 +3,7 @@
lower-level methods directly on Index and subclasses, see index_object.py,
indexing_engine.py, and index_cached.py
"""
+
from datetime import datetime
import warnings
@@ -84,9 +85,7 @@ def time_loc_slice(self, index, index_structure):
class NumericMaskedIndexing:
monotonic_list = list(range(10**6))
- non_monotonic_list = (
- list(range(50)) + [54, 53, 52, 51] + list(range(55, 10**6 - 1))
- )
+ non_monotonic_list = list(range(50)) + [54, 53, 52, 51] + list(range(55, 10**6 - 1))
params = [
("Int64", "UInt64", "Float64"),
diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py
index d5c58033c1157..ce3935d2cd0ac 100644
--- a/asv_bench/benchmarks/inference.py
+++ b/asv_bench/benchmarks/inference.py
@@ -22,23 +22,20 @@
class ToNumeric:
- params = ["ignore", "coerce"]
- param_names = ["errors"]
-
- def setup(self, errors):
+ def setup(self):
N = 10000
self.float = Series(np.random.randn(N))
self.numstr = self.float.astype("str")
self.str = Series(Index([f"i-{i}" for i in range(N)], dtype=object))
- def time_from_float(self, errors):
- to_numeric(self.float, errors=errors)
+ def time_from_float(self):
+ to_numeric(self.float, errors="coerce")
- def time_from_numeric_str(self, errors):
- to_numeric(self.numstr, errors=errors)
+ def time_from_numeric_str(self):
+ to_numeric(self.numstr, errors="coerce")
- def time_from_str(self, errors):
- to_numeric(self.str, errors=errors)
+ def time_from_str(self):
+ to_numeric(self.str, errors="coerce")
class ToNumericDowncast:
@@ -187,7 +184,7 @@ def time_iso8601_tz_spaceformat(self):
def time_iso8601_infer_zero_tz_fromat(self):
# GH 41047
- to_datetime(self.strings_zero_tz, infer_datetime_format=True)
+ to_datetime(self.strings_zero_tz)
class ToDatetimeNONISO8601:
@@ -203,7 +200,7 @@ def time_same_offset(self):
to_datetime(self.same_offset)
def time_different_offset(self):
- to_datetime(self.diff_offset)
+ to_datetime(self.diff_offset, utc=True)
class ToDatetimeFormatQuarters:
@@ -234,9 +231,6 @@ def time_no_exact(self):
def time_same_offset(self):
to_datetime(self.same_offset, format="%m/%d/%Y %H:%M:%S.%f%z")
- def time_different_offset(self):
- to_datetime(self.diff_offset, format="%m/%d/%Y %H:%M:%S.%f%z")
-
def time_same_offset_to_utc(self):
to_datetime(self.same_offset, format="%m/%d/%Y %H:%M:%S.%f%z", utc=True)
@@ -271,16 +265,6 @@ def time_dup_string_tzoffset_dates(self, cache):
to_datetime(self.dup_string_with_tz, cache=cache)
-# GH 43901
-class ToDatetimeInferDatetimeFormat:
- def setup(self):
- rng = date_range(start="1/1/2000", periods=100000, freq="h")
- self.strings = rng.strftime("%Y-%m-%d %H:%M:%S").tolist()
-
- def time_infer_datetime_format(self):
- to_datetime(self.strings, infer_datetime_format=True)
-
-
class ToTimedelta:
def setup(self):
self.ints = np.random.randint(0, 60, size=10000)
@@ -301,16 +285,13 @@ def time_convert_string_seconds(self):
class ToTimedeltaErrors:
- params = ["coerce", "ignore"]
- param_names = ["errors"]
-
- def setup(self, errors):
+ def setup(self):
ints = np.random.randint(0, 60, size=10000)
self.arr = [f"{i} days" for i in ints]
self.arr[-1] = "apple"
- def time_convert(self, errors):
- to_timedelta(self.arr, errors=errors)
+ def time_convert(self):
+ to_timedelta(self.arr, errors="coerce")
from .pandas_vb_common import setup # noqa: F401 isort:skip
diff --git a/asv_bench/benchmarks/io/csv.py b/asv_bench/benchmarks/io/csv.py
index 9ac83db4f85b9..dae6107db4d92 100644
--- a/asv_bench/benchmarks/io/csv.py
+++ b/asv_bench/benchmarks/io/csv.py
@@ -408,6 +408,9 @@ def time_read_stringcsv(self, engine):
def time_read_bytescsv(self, engine):
read_csv(self.data(self.BytesIO_input), engine=engine)
+ def peakmem_read_csv(self, engine):
+ read_csv(self.data(self.BytesIO_input), engine=engine)
+
class ReadCSVCategorical(BaseIO):
fname = "__test__.csv"
diff --git a/asv_bench/benchmarks/io/hdf.py b/asv_bench/benchmarks/io/hdf.py
index acf0ec4b9d359..2eb4c8c7f674b 100644
--- a/asv_bench/benchmarks/io/hdf.py
+++ b/asv_bench/benchmarks/io/hdf.py
@@ -123,12 +123,12 @@ def setup(self, format):
index=date_range("20000101", periods=N, freq="h"),
)
self.df["object"] = Index([f"i-{i}" for i in range(N)], dtype=object)
- self.df.to_hdf(self.fname, "df", format=format)
+ self.df.to_hdf(self.fname, key="df", format=format)
# Numeric df
self.df1 = self.df.copy()
self.df1 = self.df1.reset_index()
- self.df1.to_hdf(self.fname, "df1", format=format)
+ self.df1.to_hdf(self.fname, key="df1", format=format)
def time_read_hdf(self, format):
read_hdf(self.fname, "df")
@@ -137,7 +137,7 @@ def peakmem_read_hdf(self, format):
read_hdf(self.fname, "df")
def time_write_hdf(self, format):
- self.df.to_hdf(self.fname, "df", format=format)
+ self.df.to_hdf(self.fname, key="df", format=format)
from ..pandas_vb_common import setup # noqa: F401 isort:skip
diff --git a/asv_bench/benchmarks/io/style.py b/asv_bench/benchmarks/io/style.py
index af9eef337e78e..24fd8a0d20aba 100644
--- a/asv_bench/benchmarks/io/style.py
+++ b/asv_bench/benchmarks/io/style.py
@@ -76,7 +76,8 @@ def _style_format(self):
# apply a formatting function
# subset is flexible but hinders vectorised solutions
self.st = self.df.style.format(
- "{:,.3f}", subset=IndexSlice["row_1":f"row_{ir}", "float_1":f"float_{ic}"]
+ "{:,.3f}",
+ subset=IndexSlice["row_1" : f"row_{ir}", "float_1" : f"float_{ic}"],
)
def _style_apply_format_hide(self):
diff --git a/asv_bench/benchmarks/join_merge.py b/asv_bench/benchmarks/join_merge.py
index ce64304731116..a6c6990892d38 100644
--- a/asv_bench/benchmarks/join_merge.py
+++ b/asv_bench/benchmarks/join_merge.py
@@ -328,6 +328,23 @@ def time_i8merge(self, how):
merge(self.left, self.right, how=how)
+class UniqueMerge:
+ params = [4_000_000, 1_000_000]
+ param_names = ["unique_elements"]
+
+ def setup(self, unique_elements):
+ N = 1_000_000
+ self.left = DataFrame({"a": np.random.randint(1, unique_elements, (N,))})
+ self.right = DataFrame({"a": np.random.randint(1, unique_elements, (N,))})
+ uniques = self.right.a.drop_duplicates()
+ self.right["a"] = concat(
+ [uniques, Series(np.arange(0, -(N - len(uniques)), -1))], ignore_index=True
+ )
+
+ def time_unique_merge(self, unique_elements):
+ merge(self.left, self.right, how="inner")
+
+
class MergeDatetime:
params = [
[
diff --git a/asv_bench/benchmarks/libs.py b/asv_bench/benchmarks/libs.py
index 3419163bcfe09..7da2d27d98dbb 100644
--- a/asv_bench/benchmarks/libs.py
+++ b/asv_bench/benchmarks/libs.py
@@ -5,6 +5,7 @@
If a PR does not edit anything in _libs/, then it is unlikely that the
benchmarks will be affected.
"""
+
import numpy as np
from pandas._libs.lib import (
diff --git a/asv_bench/benchmarks/multiindex_object.py b/asv_bench/benchmarks/multiindex_object.py
index 54788d41d83fe..0a588c9a2e22e 100644
--- a/asv_bench/benchmarks/multiindex_object.py
+++ b/asv_bench/benchmarks/multiindex_object.py
@@ -223,14 +223,20 @@ def time_categorical_level(self):
class Equals:
def setup(self):
- idx_large_fast = RangeIndex(100000)
- idx_small_slow = date_range(start="1/1/2012", periods=1)
- self.mi_large_slow = MultiIndex.from_product([idx_large_fast, idx_small_slow])
-
+ self.mi = MultiIndex.from_product(
+ [
+ date_range("2000-01-01", periods=1000),
+ RangeIndex(1000),
+ ]
+ )
+ self.mi_deepcopy = self.mi.copy(deep=True)
self.idx_non_object = RangeIndex(1)
+ def time_equals_deepcopy(self):
+ self.mi.equals(self.mi_deepcopy)
+
def time_equals_non_object_index(self):
- self.mi_large_slow.equals(self.idx_non_object)
+ self.mi.equals(self.idx_non_object)
class SetOperations:
diff --git a/asv_bench/benchmarks/package.py b/asv_bench/benchmarks/package.py
index 257c82cba8878..f8b51a523dab8 100644
--- a/asv_bench/benchmarks/package.py
+++ b/asv_bench/benchmarks/package.py
@@ -1,6 +1,7 @@
"""
Benchmarks for pandas at the package-level.
"""
+
import subprocess
import sys
diff --git a/asv_bench/benchmarks/period.py b/asv_bench/benchmarks/period.py
index ccd86cae06d58..3b8b60e790380 100644
--- a/asv_bench/benchmarks/period.py
+++ b/asv_bench/benchmarks/period.py
@@ -2,6 +2,7 @@
Period benchmarks with non-tslibs dependencies. See
benchmarks.tslibs.period for benchmarks that rely only on tslibs.
"""
+
from pandas import (
DataFrame,
Period,
diff --git a/asv_bench/benchmarks/stat_ops.py b/asv_bench/benchmarks/stat_ops.py
index 89bda81ccf08c..8913293dfa20e 100644
--- a/asv_bench/benchmarks/stat_ops.py
+++ b/asv_bench/benchmarks/stat_ops.py
@@ -33,6 +33,7 @@ def setup(self, op, axis):
("median", 1),
("median", None),
("std", 1),
+ ("std", None),
)
):
# Skipping cases where datetime aggregations are not implemented
diff --git a/asv_bench/benchmarks/strftime.py b/asv_bench/benchmarks/strftime.py
index 47f25b331ab9b..5f2832e361eb5 100644
--- a/asv_bench/benchmarks/strftime.py
+++ b/asv_bench/benchmarks/strftime.py
@@ -79,12 +79,6 @@ def time_frame_period_formatting_default(self, nobs, freq):
def time_frame_period_formatting_default_explicit(self, nobs, freq):
self.data["p"].dt.strftime(date_format=self.default_fmt)
- def time_frame_period_formatting_index_default(self, nobs, freq):
- self.data.index.format()
-
- def time_frame_period_formatting_index_default_explicit(self, nobs, freq):
- self.data.index.format(self.default_fmt)
-
def time_frame_period_formatting_custom(self, nobs, freq):
self.data["p"].dt.strftime(date_format="%Y-%m-%d --- %H:%M:%S")
diff --git a/asv_bench/benchmarks/strings.py b/asv_bench/benchmarks/strings.py
index 1f4a104255057..467fab857d306 100644
--- a/asv_bench/benchmarks/strings.py
+++ b/asv_bench/benchmarks/strings.py
@@ -19,10 +19,11 @@ class Dtypes:
def setup(self, dtype):
try:
self.s = Series(
- Index([f"i-{i}" for i in range(10000)], dtype=object), dtype=dtype
+ Index([f"i-{i}" for i in range(10000)], dtype=object)._values,
+ dtype=dtype,
)
- except ImportError:
- raise NotImplementedError
+ except ImportError as err:
+ raise NotImplementedError from err
class Construction:
diff --git a/asv_bench/benchmarks/timeseries.py b/asv_bench/benchmarks/timeseries.py
index 8e1deb99a66a4..06f488f7baaaf 100644
--- a/asv_bench/benchmarks/timeseries.py
+++ b/asv_bench/benchmarks/timeseries.py
@@ -183,7 +183,7 @@ def setup(self):
self.dt_ts = Series(5, rng3, dtype="datetime64[ns]")
def time_resample(self):
- self.dt_ts.resample("1S").last()
+ self.dt_ts.resample("1s").last()
class AsOf:
diff --git a/asv_bench/benchmarks/tslibs/offsets.py b/asv_bench/benchmarks/tslibs/offsets.py
index 1f48ec504acf1..55bd3c31c055c 100644
--- a/asv_bench/benchmarks/tslibs/offsets.py
+++ b/asv_bench/benchmarks/tslibs/offsets.py
@@ -2,6 +2,7 @@
offsets benchmarks that rely only on tslibs. See benchmarks.offset for
offsets benchmarks that rely on other parts of pandas.
"""
+
from datetime import datetime
import numpy as np
diff --git a/asv_bench/benchmarks/tslibs/resolution.py b/asv_bench/benchmarks/tslibs/resolution.py
index 44f288c7de216..6317d299379d3 100644
--- a/asv_bench/benchmarks/tslibs/resolution.py
+++ b/asv_bench/benchmarks/tslibs/resolution.py
@@ -17,6 +17,7 @@
df.loc[key] = (val.average, val.stdev)
"""
+
import numpy as np
try:
diff --git a/asv_bench/benchmarks/tslibs/timedelta.py b/asv_bench/benchmarks/tslibs/timedelta.py
index 2daf1861eb80a..dcc73aefc6c7a 100644
--- a/asv_bench/benchmarks/tslibs/timedelta.py
+++ b/asv_bench/benchmarks/tslibs/timedelta.py
@@ -2,6 +2,7 @@
Timedelta benchmarks that rely only on tslibs. See benchmarks.timedeltas for
Timedelta benchmarks that rely on other parts of pandas.
"""
+
import datetime
import numpy as np
diff --git a/asv_bench/benchmarks/tslibs/tslib.py b/asv_bench/benchmarks/tslibs/tslib.py
index 97ec80201dd16..4a011d4bb3f06 100644
--- a/asv_bench/benchmarks/tslibs/tslib.py
+++ b/asv_bench/benchmarks/tslibs/tslib.py
@@ -15,6 +15,7 @@
val = %timeit -o tr.time_ints_to_pydatetime(box, size, tz)
df.loc[key] = (val.average, val.stdev)
"""
+
from datetime import (
timedelta,
timezone,
diff --git a/ci/code_checks.sh b/ci/code_checks.sh
index e41f625e583c0..0c4e6641444f1 100755
--- a/ci/code_checks.sh
+++ b/ci/code_checks.sh
@@ -16,12 +16,18 @@
set -uo pipefail
-[[ -z "$1" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "single-docs" || "$1" == "notebooks" ]] || \
+if [[ -v 1 ]]; then
+ CHECK=$1
+else
+ # script will fail if it uses an unset variable (i.e. $1 is not provided)
+ CHECK=""
+fi
+
+[[ -z "$CHECK" || "$CHECK" == "code" || "$CHECK" == "doctests" || "$CHECK" == "docstrings" || "$CHECK" == "single-docs" || "$CHECK" == "notebooks" ]] || \
{ echo "Unknown command $1. Usage: $0 [code|doctests|docstrings|single-docs|notebooks]"; exit 9999; }
BASE_DIR="$(dirname $0)/.."
RET=0
-CHECK=$1
### CODE ###
if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then
@@ -59,110 +65,1204 @@ fi
### DOCSTRINGS ###
if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
- MSG='Validate docstrings (EX01, EX02, EX04, GL01, GL02, GL03, GL04, GL05, GL06, GL07, GL09, GL10, PR03, PR04, PR05, PR06, PR08, PR09, PR10, RT01, RT02, RT04, RT05, SA02, SA03, SA04, SS01, SS02, SS03, SS04, SS05, SS06)' ; echo $MSG
- $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX01,EX02,EX04,GL01,GL02,GL03,GL04,GL05,GL06,GL07,GL09,GL10,PR03,PR04,PR05,PR06,PR08,PR09,PR10,RT01,RT02,RT04,RT05,SA02,SA03,SA04,SS01,SS02,SS03,SS04,SS05,SS06
- RET=$(($RET + $?)) ; echo $MSG "DONE"
+ MSG='Validate Docstrings' ; echo $MSG
+ $BASE_DIR/scripts/validate_docstrings.py \
+ --format=actions \
+ -i ES01 `# For now it is ok if docstrings are missing the extended summary` \
+ -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
+ -i "pandas.Categorical.__array__ SA01" \
+ -i "pandas.Categorical.codes SA01" \
+ -i "pandas.Categorical.dtype SA01" \
+ -i "pandas.Categorical.from_codes SA01" \
+ -i "pandas.Categorical.ordered SA01" \
+ -i "pandas.CategoricalDtype.categories SA01" \
+ -i "pandas.CategoricalDtype.ordered SA01" \
+ -i "pandas.CategoricalIndex.codes SA01" \
+ -i "pandas.CategoricalIndex.ordered SA01" \
+ -i "pandas.DataFrame.__dataframe__ SA01" \
+ -i "pandas.DataFrame.__iter__ SA01" \
+ -i "pandas.DataFrame.assign SA01" \
+ -i "pandas.DataFrame.at_time PR01" \
+ -i "pandas.DataFrame.axes SA01" \
+ -i "pandas.DataFrame.backfill PR01,SA01" \
+ -i "pandas.DataFrame.bfill SA01" \
+ -i "pandas.DataFrame.columns SA01" \
+ -i "pandas.DataFrame.copy SA01" \
+ -i "pandas.DataFrame.droplevel SA01" \
+ -i "pandas.DataFrame.dtypes SA01" \
+ -i "pandas.DataFrame.ffill SA01" \
+ -i "pandas.DataFrame.first_valid_index SA01" \
+ -i "pandas.DataFrame.get SA01" \
+ -i "pandas.DataFrame.hist RT03" \
+ -i "pandas.DataFrame.infer_objects RT03" \
+ -i "pandas.DataFrame.keys SA01" \
+ -i "pandas.DataFrame.kurt RT03,SA01" \
+ -i "pandas.DataFrame.kurtosis RT03,SA01" \
+ -i "pandas.DataFrame.last_valid_index SA01" \
+ -i "pandas.DataFrame.mask RT03" \
+ -i "pandas.DataFrame.max RT03" \
+ -i "pandas.DataFrame.mean RT03,SA01" \
+ -i "pandas.DataFrame.median RT03,SA01" \
+ -i "pandas.DataFrame.min RT03" \
+ -i "pandas.DataFrame.pad PR01,SA01" \
+ -i "pandas.DataFrame.plot PR02,SA01" \
+ -i "pandas.DataFrame.pop SA01" \
+ -i "pandas.DataFrame.prod RT03" \
+ -i "pandas.DataFrame.product RT03" \
+ -i "pandas.DataFrame.reorder_levels SA01" \
+ -i "pandas.DataFrame.sem PR01,RT03,SA01" \
+ -i "pandas.DataFrame.skew RT03,SA01" \
+ -i "pandas.DataFrame.sparse PR01,SA01" \
+ -i "pandas.DataFrame.sparse.density SA01" \
+ -i "pandas.DataFrame.sparse.from_spmatrix SA01" \
+ -i "pandas.DataFrame.sparse.to_coo SA01" \
+ -i "pandas.DataFrame.sparse.to_dense SA01" \
+ -i "pandas.DataFrame.std PR01,RT03,SA01" \
+ -i "pandas.DataFrame.sum RT03" \
+ -i "pandas.DataFrame.swapaxes PR01,SA01" \
+ -i "pandas.DataFrame.swaplevel SA01" \
+ -i "pandas.DataFrame.to_feather SA01" \
+ -i "pandas.DataFrame.to_markdown SA01" \
+ -i "pandas.DataFrame.to_parquet RT03" \
+ -i "pandas.DataFrame.to_period SA01" \
+ -i "pandas.DataFrame.to_timestamp SA01" \
+ -i "pandas.DataFrame.tz_convert SA01" \
+ -i "pandas.DataFrame.tz_localize SA01" \
+ -i "pandas.DataFrame.unstack RT03" \
+ -i "pandas.DataFrame.value_counts RT03" \
+ -i "pandas.DataFrame.var PR01,RT03,SA01" \
+ -i "pandas.DataFrame.where RT03" \
+ -i "pandas.DatetimeIndex.ceil SA01" \
+ -i "pandas.DatetimeIndex.date SA01" \
+ -i "pandas.DatetimeIndex.day SA01" \
+ -i "pandas.DatetimeIndex.day_name SA01" \
+ -i "pandas.DatetimeIndex.day_of_year SA01" \
+ -i "pandas.DatetimeIndex.dayofyear SA01" \
+ -i "pandas.DatetimeIndex.floor SA01" \
+ -i "pandas.DatetimeIndex.freqstr SA01" \
+ -i "pandas.DatetimeIndex.hour SA01" \
+ -i "pandas.DatetimeIndex.indexer_at_time PR01,RT03" \
+ -i "pandas.DatetimeIndex.indexer_between_time RT03" \
+ -i "pandas.DatetimeIndex.inferred_freq SA01" \
+ -i "pandas.DatetimeIndex.is_leap_year SA01" \
+ -i "pandas.DatetimeIndex.microsecond SA01" \
+ -i "pandas.DatetimeIndex.minute SA01" \
+ -i "pandas.DatetimeIndex.month SA01" \
+ -i "pandas.DatetimeIndex.month_name SA01" \
+ -i "pandas.DatetimeIndex.nanosecond SA01" \
+ -i "pandas.DatetimeIndex.quarter SA01" \
+ -i "pandas.DatetimeIndex.round SA01" \
+ -i "pandas.DatetimeIndex.second SA01" \
+ -i "pandas.DatetimeIndex.snap PR01,RT03,SA01" \
+ -i "pandas.DatetimeIndex.std PR01,RT03" \
+ -i "pandas.DatetimeIndex.time SA01" \
+ -i "pandas.DatetimeIndex.timetz SA01" \
+ -i "pandas.DatetimeIndex.to_period RT03" \
+ -i "pandas.DatetimeIndex.to_pydatetime RT03,SA01" \
+ -i "pandas.DatetimeIndex.tz SA01" \
+ -i "pandas.DatetimeIndex.tz_convert RT03" \
+ -i "pandas.DatetimeIndex.year SA01" \
+ -i "pandas.DatetimeTZDtype SA01" \
+ -i "pandas.DatetimeTZDtype.tz SA01" \
+ -i "pandas.DatetimeTZDtype.unit SA01" \
+ -i "pandas.ExcelFile PR01,SA01" \
+ -i "pandas.ExcelFile.parse PR01,SA01" \
+ -i "pandas.ExcelWriter SA01" \
+ -i "pandas.Float32Dtype SA01" \
+ -i "pandas.Float64Dtype SA01" \
+ -i "pandas.Grouper PR02,SA01" \
+ -i "pandas.HDFStore.append PR01,SA01" \
+ -i "pandas.HDFStore.get SA01" \
+ -i "pandas.HDFStore.groups SA01" \
+ -i "pandas.HDFStore.info RT03,SA01" \
+ -i "pandas.HDFStore.keys SA01" \
+ -i "pandas.HDFStore.put PR01,SA01" \
+ -i "pandas.HDFStore.select SA01" \
+ -i "pandas.HDFStore.walk SA01" \
+ -i "pandas.Index PR07" \
+ -i "pandas.Index.T SA01" \
+ -i "pandas.Index.append PR07,RT03,SA01" \
+ -i "pandas.Index.astype SA01" \
+ -i "pandas.Index.copy PR07,SA01" \
+ -i "pandas.Index.difference PR07,RT03,SA01" \
+ -i "pandas.Index.drop PR07,SA01" \
+ -i "pandas.Index.drop_duplicates RT03" \
+ -i "pandas.Index.droplevel RT03,SA01" \
+ -i "pandas.Index.dropna RT03,SA01" \
+ -i "pandas.Index.dtype SA01" \
+ -i "pandas.Index.duplicated RT03" \
+ -i "pandas.Index.empty GL08" \
+ -i "pandas.Index.equals SA01" \
+ -i "pandas.Index.fillna RT03" \
+ -i "pandas.Index.get_indexer PR07,SA01" \
+ -i "pandas.Index.get_indexer_for PR01,SA01" \
+ -i "pandas.Index.get_indexer_non_unique PR07,SA01" \
+ -i "pandas.Index.get_loc PR07,RT03,SA01" \
+ -i "pandas.Index.get_slice_bound PR07" \
+ -i "pandas.Index.hasnans SA01" \
+ -i "pandas.Index.identical PR01,SA01" \
+ -i "pandas.Index.inferred_type SA01" \
+ -i "pandas.Index.insert PR07,RT03,SA01" \
+ -i "pandas.Index.intersection PR07,RT03,SA01" \
+ -i "pandas.Index.item SA01" \
+ -i "pandas.Index.join PR07,RT03,SA01" \
+ -i "pandas.Index.map SA01" \
+ -i "pandas.Index.memory_usage RT03" \
+ -i "pandas.Index.name SA01" \
+ -i "pandas.Index.names GL08" \
+ -i "pandas.Index.nbytes SA01" \
+ -i "pandas.Index.ndim SA01" \
+ -i "pandas.Index.nunique RT03" \
+ -i "pandas.Index.putmask PR01,RT03" \
+ -i "pandas.Index.ravel PR01,RT03" \
+ -i "pandas.Index.reindex PR07" \
+ -i "pandas.Index.shape SA01" \
+ -i "pandas.Index.size SA01" \
+ -i "pandas.Index.slice_indexer PR07,RT03,SA01" \
+ -i "pandas.Index.slice_locs RT03" \
+ -i "pandas.Index.str PR01,SA01" \
+ -i "pandas.Index.symmetric_difference PR07,RT03,SA01" \
+ -i "pandas.Index.take PR01,PR07" \
+ -i "pandas.Index.to_list RT03" \
+ -i "pandas.Index.union PR07,RT03,SA01" \
+ -i "pandas.Index.unique RT03" \
+ -i "pandas.Index.value_counts RT03" \
+ -i "pandas.Index.view GL08" \
+ -i "pandas.Int16Dtype SA01" \
+ -i "pandas.Int32Dtype SA01" \
+ -i "pandas.Int64Dtype SA01" \
+ -i "pandas.Int8Dtype SA01" \
+ -i "pandas.Interval PR02" \
+ -i "pandas.Interval.closed SA01" \
+ -i "pandas.Interval.left SA01" \
+ -i "pandas.Interval.mid SA01" \
+ -i "pandas.Interval.right SA01" \
+ -i "pandas.IntervalDtype PR01,SA01" \
+ -i "pandas.IntervalDtype.subtype SA01" \
+ -i "pandas.IntervalIndex.closed SA01" \
+ -i "pandas.IntervalIndex.contains RT03" \
+ -i "pandas.IntervalIndex.get_indexer PR07,SA01" \
+ -i "pandas.IntervalIndex.get_loc PR07,RT03,SA01" \
+ -i "pandas.IntervalIndex.is_non_overlapping_monotonic SA01" \
+ -i "pandas.IntervalIndex.left GL08" \
+ -i "pandas.IntervalIndex.length GL08" \
+ -i "pandas.IntervalIndex.mid GL08" \
+ -i "pandas.IntervalIndex.right GL08" \
+ -i "pandas.IntervalIndex.set_closed RT03,SA01" \
+ -i "pandas.IntervalIndex.to_tuples RT03,SA01" \
+ -i "pandas.MultiIndex PR01" \
+ -i "pandas.MultiIndex.append PR07,SA01" \
+ -i "pandas.MultiIndex.copy PR07,RT03,SA01" \
+ -i "pandas.MultiIndex.drop PR07,RT03,SA01" \
+ -i "pandas.MultiIndex.droplevel RT03,SA01" \
+ -i "pandas.MultiIndex.dtypes SA01" \
+ -i "pandas.MultiIndex.get_indexer PR07,SA01" \
+ -i "pandas.MultiIndex.get_level_values SA01" \
+ -i "pandas.MultiIndex.get_loc PR07" \
+ -i "pandas.MultiIndex.get_loc_level PR07" \
+ -i "pandas.MultiIndex.levels SA01" \
+ -i "pandas.MultiIndex.levshape SA01" \
+ -i "pandas.MultiIndex.names SA01" \
+ -i "pandas.MultiIndex.nlevels SA01" \
+ -i "pandas.MultiIndex.remove_unused_levels RT03,SA01" \
+ -i "pandas.MultiIndex.reorder_levels RT03,SA01" \
+ -i "pandas.MultiIndex.set_codes SA01" \
+ -i "pandas.MultiIndex.set_levels RT03,SA01" \
+ -i "pandas.MultiIndex.sortlevel PR07,SA01" \
+ -i "pandas.MultiIndex.to_frame RT03" \
+ -i "pandas.MultiIndex.truncate SA01" \
+ -i "pandas.NA SA01" \
+ -i "pandas.NaT SA01" \
+ -i "pandas.NamedAgg SA01" \
+ -i "pandas.Period SA01" \
+ -i "pandas.Period.asfreq SA01" \
+ -i "pandas.Period.freq GL08" \
+ -i "pandas.Period.freqstr SA01" \
+ -i "pandas.Period.is_leap_year SA01" \
+ -i "pandas.Period.month SA01" \
+ -i "pandas.Period.now SA01" \
+ -i "pandas.Period.ordinal GL08" \
+ -i "pandas.Period.quarter SA01" \
+ -i "pandas.Period.strftime PR01,SA01" \
+ -i "pandas.Period.to_timestamp SA01" \
+ -i "pandas.Period.year SA01" \
+ -i "pandas.PeriodDtype SA01" \
+ -i "pandas.PeriodDtype.freq SA01" \
+ -i "pandas.PeriodIndex.day SA01" \
+ -i "pandas.PeriodIndex.day_of_week SA01" \
+ -i "pandas.PeriodIndex.day_of_year SA01" \
+ -i "pandas.PeriodIndex.dayofweek SA01" \
+ -i "pandas.PeriodIndex.dayofyear SA01" \
+ -i "pandas.PeriodIndex.days_in_month SA01" \
+ -i "pandas.PeriodIndex.daysinmonth SA01" \
+ -i "pandas.PeriodIndex.freqstr SA01" \
+ -i "pandas.PeriodIndex.from_fields PR07,SA01" \
+ -i "pandas.PeriodIndex.from_ordinals SA01" \
+ -i "pandas.PeriodIndex.hour SA01" \
+ -i "pandas.PeriodIndex.is_leap_year SA01" \
+ -i "pandas.PeriodIndex.minute SA01" \
+ -i "pandas.PeriodIndex.month SA01" \
+ -i "pandas.PeriodIndex.quarter SA01" \
+ -i "pandas.PeriodIndex.qyear GL08" \
+ -i "pandas.PeriodIndex.second SA01" \
+ -i "pandas.PeriodIndex.to_timestamp RT03,SA01" \
+ -i "pandas.PeriodIndex.week SA01" \
+ -i "pandas.PeriodIndex.weekday SA01" \
+ -i "pandas.PeriodIndex.weekofyear SA01" \
+ -i "pandas.PeriodIndex.year SA01" \
+ -i "pandas.RangeIndex PR07" \
+ -i "pandas.RangeIndex.from_range PR01,SA01" \
+ -i "pandas.RangeIndex.start SA01" \
+ -i "pandas.RangeIndex.step SA01" \
+ -i "pandas.RangeIndex.stop SA01" \
+ -i "pandas.Series SA01" \
+ -i "pandas.Series.T SA01" \
+ -i "pandas.Series.__iter__ RT03,SA01" \
+ -i "pandas.Series.add PR07" \
+ -i "pandas.Series.at_time PR01" \
+ -i "pandas.Series.backfill PR01,SA01" \
+ -i "pandas.Series.bfill SA01" \
+ -i "pandas.Series.case_when RT03" \
+ -i "pandas.Series.cat PR07,SA01" \
+ -i "pandas.Series.cat.add_categories PR01,PR02" \
+ -i "pandas.Series.cat.as_ordered PR01" \
+ -i "pandas.Series.cat.as_unordered PR01" \
+ -i "pandas.Series.cat.codes SA01" \
+ -i "pandas.Series.cat.ordered SA01" \
+ -i "pandas.Series.cat.remove_categories PR01,PR02" \
+ -i "pandas.Series.cat.remove_unused_categories PR01" \
+ -i "pandas.Series.cat.rename_categories PR01,PR02" \
+ -i "pandas.Series.cat.reorder_categories PR01,PR02" \
+ -i "pandas.Series.cat.set_categories PR01,PR02" \
+ -i "pandas.Series.copy SA01" \
+ -i "pandas.Series.div PR07" \
+ -i "pandas.Series.droplevel SA01" \
+ -i "pandas.Series.dt.as_unit PR01,PR02" \
+ -i "pandas.Series.dt.ceil PR01,PR02,SA01" \
+ -i "pandas.Series.dt.components SA01" \
+ -i "pandas.Series.dt.date SA01" \
+ -i "pandas.Series.dt.day SA01" \
+ -i "pandas.Series.dt.day_name PR01,PR02,SA01" \
+ -i "pandas.Series.dt.day_of_year SA01" \
+ -i "pandas.Series.dt.dayofyear SA01" \
+ -i "pandas.Series.dt.days SA01" \
+ -i "pandas.Series.dt.days_in_month SA01" \
+ -i "pandas.Series.dt.daysinmonth SA01" \
+ -i "pandas.Series.dt.floor PR01,PR02,SA01" \
+ -i "pandas.Series.dt.freq GL08" \
+ -i "pandas.Series.dt.hour SA01" \
+ -i "pandas.Series.dt.is_leap_year SA01" \
+ -i "pandas.Series.dt.microsecond SA01" \
+ -i "pandas.Series.dt.microseconds SA01" \
+ -i "pandas.Series.dt.minute SA01" \
+ -i "pandas.Series.dt.month SA01" \
+ -i "pandas.Series.dt.month_name PR01,PR02,SA01" \
+ -i "pandas.Series.dt.nanosecond SA01" \
+ -i "pandas.Series.dt.nanoseconds SA01" \
+ -i "pandas.Series.dt.normalize PR01" \
+ -i "pandas.Series.dt.quarter SA01" \
+ -i "pandas.Series.dt.qyear GL08" \
+ -i "pandas.Series.dt.round PR01,PR02,SA01" \
+ -i "pandas.Series.dt.second SA01" \
+ -i "pandas.Series.dt.seconds SA01" \
+ -i "pandas.Series.dt.strftime PR01,PR02" \
+ -i "pandas.Series.dt.time SA01" \
+ -i "pandas.Series.dt.timetz SA01" \
+ -i "pandas.Series.dt.to_period PR01,PR02,RT03" \
+ -i "pandas.Series.dt.total_seconds PR01" \
+ -i "pandas.Series.dt.tz SA01" \
+ -i "pandas.Series.dt.tz_convert PR01,PR02,RT03" \
+ -i "pandas.Series.dt.tz_localize PR01,PR02" \
+ -i "pandas.Series.dt.unit GL08" \
+ -i "pandas.Series.dt.year SA01" \
+ -i "pandas.Series.dtype SA01" \
+ -i "pandas.Series.dtypes SA01" \
+ -i "pandas.Series.empty GL08" \
+ -i "pandas.Series.eq PR07,SA01" \
+ -i "pandas.Series.ffill SA01" \
+ -i "pandas.Series.first_valid_index SA01" \
+ -i "pandas.Series.floordiv PR07" \
+ -i "pandas.Series.ge PR07,SA01" \
+ -i "pandas.Series.get SA01" \
+ -i "pandas.Series.gt PR07,SA01" \
+ -i "pandas.Series.hasnans SA01" \
+ -i "pandas.Series.infer_objects RT03" \
+ -i "pandas.Series.is_monotonic_decreasing SA01" \
+ -i "pandas.Series.is_monotonic_increasing SA01" \
+ -i "pandas.Series.is_unique SA01" \
+ -i "pandas.Series.item SA01" \
+ -i "pandas.Series.keys SA01" \
+ -i "pandas.Series.kurt RT03,SA01" \
+ -i "pandas.Series.kurtosis RT03,SA01" \
+ -i "pandas.Series.last_valid_index SA01" \
+ -i "pandas.Series.le PR07,SA01" \
+ -i "pandas.Series.list.__getitem__ SA01" \
+ -i "pandas.Series.list.flatten SA01" \
+ -i "pandas.Series.list.len SA01" \
+ -i "pandas.Series.lt PR07,SA01" \
+ -i "pandas.Series.mask RT03" \
+ -i "pandas.Series.max RT03" \
+ -i "pandas.Series.mean RT03,SA01" \
+ -i "pandas.Series.median RT03,SA01" \
+ -i "pandas.Series.min RT03" \
+ -i "pandas.Series.mod PR07" \
+ -i "pandas.Series.mode SA01" \
+ -i "pandas.Series.mul PR07" \
+ -i "pandas.Series.nbytes SA01" \
+ -i "pandas.Series.ndim SA01" \
+ -i "pandas.Series.ne PR07,SA01" \
+ -i "pandas.Series.nunique RT03" \
+ -i "pandas.Series.pad PR01,SA01" \
+ -i "pandas.Series.plot PR02,SA01" \
+ -i "pandas.Series.pop RT03,SA01" \
+ -i "pandas.Series.pow PR07" \
+ -i "pandas.Series.prod RT03" \
+ -i "pandas.Series.product RT03" \
+ -i "pandas.Series.radd PR07" \
+ -i "pandas.Series.rdiv PR07" \
+ -i "pandas.Series.reorder_levels RT03,SA01" \
+ -i "pandas.Series.rfloordiv PR07" \
+ -i "pandas.Series.rmod PR07" \
+ -i "pandas.Series.rmul PR07" \
+ -i "pandas.Series.rpow PR07" \
+ -i "pandas.Series.rsub PR07" \
+ -i "pandas.Series.rtruediv PR07" \
+ -i "pandas.Series.sem PR01,RT03,SA01" \
+ -i "pandas.Series.shape SA01" \
+ -i "pandas.Series.size SA01" \
+ -i "pandas.Series.skew RT03,SA01" \
+ -i "pandas.Series.sparse PR01,SA01" \
+ -i "pandas.Series.sparse.density SA01" \
+ -i "pandas.Series.sparse.fill_value SA01" \
+ -i "pandas.Series.sparse.from_coo PR07,SA01" \
+ -i "pandas.Series.sparse.npoints SA01" \
+ -i "pandas.Series.sparse.sp_values SA01" \
+ -i "pandas.Series.sparse.to_coo PR07,RT03,SA01" \
+ -i "pandas.Series.std PR01,RT03,SA01" \
+ -i "pandas.Series.str PR01,SA01" \
+ -i "pandas.Series.str.capitalize RT03" \
+ -i "pandas.Series.str.casefold RT03" \
+ -i "pandas.Series.str.center RT03,SA01" \
+ -i "pandas.Series.str.decode PR07,RT03,SA01" \
+ -i "pandas.Series.str.encode PR07,RT03,SA01" \
+ -i "pandas.Series.str.find RT03" \
+ -i "pandas.Series.str.fullmatch RT03" \
+ -i "pandas.Series.str.get RT03,SA01" \
+ -i "pandas.Series.str.index RT03" \
+ -i "pandas.Series.str.ljust RT03,SA01" \
+ -i "pandas.Series.str.lower RT03" \
+ -i "pandas.Series.str.lstrip RT03" \
+ -i "pandas.Series.str.match RT03" \
+ -i "pandas.Series.str.normalize RT03,SA01" \
+ -i "pandas.Series.str.partition RT03" \
+ -i "pandas.Series.str.repeat SA01" \
+ -i "pandas.Series.str.replace SA01" \
+ -i "pandas.Series.str.rfind RT03" \
+ -i "pandas.Series.str.rindex RT03" \
+ -i "pandas.Series.str.rjust RT03,SA01" \
+ -i "pandas.Series.str.rpartition RT03" \
+ -i "pandas.Series.str.rstrip RT03" \
+ -i "pandas.Series.str.strip RT03" \
+ -i "pandas.Series.str.swapcase RT03" \
+ -i "pandas.Series.str.title RT03" \
+ -i "pandas.Series.str.translate RT03,SA01" \
+ -i "pandas.Series.str.upper RT03" \
+ -i "pandas.Series.str.wrap RT03,SA01" \
+ -i "pandas.Series.str.zfill RT03" \
+ -i "pandas.Series.struct.dtypes SA01" \
+ -i "pandas.Series.sub PR07" \
+ -i "pandas.Series.sum RT03" \
+ -i "pandas.Series.swaplevel SA01" \
+ -i "pandas.Series.to_dict SA01" \
+ -i "pandas.Series.to_frame SA01" \
+ -i "pandas.Series.to_list RT03" \
+ -i "pandas.Series.to_markdown SA01" \
+ -i "pandas.Series.to_period SA01" \
+ -i "pandas.Series.to_string SA01" \
+ -i "pandas.Series.to_timestamp RT03,SA01" \
+ -i "pandas.Series.truediv PR07" \
+ -i "pandas.Series.tz_convert SA01" \
+ -i "pandas.Series.tz_localize SA01" \
+ -i "pandas.Series.unstack SA01" \
+ -i "pandas.Series.update PR07,SA01" \
+ -i "pandas.Series.value_counts RT03" \
+ -i "pandas.Series.var PR01,RT03,SA01" \
+ -i "pandas.Series.where RT03" \
+ -i "pandas.SparseDtype SA01" \
+ -i "pandas.Timedelta PR07,SA01" \
+ -i "pandas.Timedelta.as_unit SA01" \
+ -i "pandas.Timedelta.asm8 SA01" \
+ -i "pandas.Timedelta.ceil SA01" \
+ -i "pandas.Timedelta.components SA01" \
+ -i "pandas.Timedelta.days SA01" \
+ -i "pandas.Timedelta.floor SA01" \
+ -i "pandas.Timedelta.max PR02,PR07,SA01" \
+ -i "pandas.Timedelta.min PR02,PR07,SA01" \
+ -i "pandas.Timedelta.resolution PR02,PR07,SA01" \
+ -i "pandas.Timedelta.round SA01" \
+ -i "pandas.Timedelta.to_numpy PR01" \
+ -i "pandas.Timedelta.to_timedelta64 SA01" \
+ -i "pandas.Timedelta.total_seconds SA01" \
+ -i "pandas.Timedelta.view SA01" \
+ -i "pandas.TimedeltaIndex.as_unit RT03,SA01" \
+ -i "pandas.TimedeltaIndex.ceil SA01" \
+ -i "pandas.TimedeltaIndex.components SA01" \
+ -i "pandas.TimedeltaIndex.days SA01" \
+ -i "pandas.TimedeltaIndex.floor SA01" \
+ -i "pandas.TimedeltaIndex.inferred_freq SA01" \
+ -i "pandas.TimedeltaIndex.microseconds SA01" \
+ -i "pandas.TimedeltaIndex.nanoseconds SA01" \
+ -i "pandas.TimedeltaIndex.round SA01" \
+ -i "pandas.TimedeltaIndex.seconds SA01" \
+ -i "pandas.TimedeltaIndex.to_pytimedelta RT03,SA01" \
+ -i "pandas.Timestamp PR07,SA01" \
+ -i "pandas.Timestamp.as_unit SA01" \
+ -i "pandas.Timestamp.asm8 SA01" \
+ -i "pandas.Timestamp.astimezone SA01" \
+ -i "pandas.Timestamp.ceil SA01" \
+ -i "pandas.Timestamp.combine PR01,SA01" \
+ -i "pandas.Timestamp.ctime SA01" \
+ -i "pandas.Timestamp.date SA01" \
+ -i "pandas.Timestamp.day GL08" \
+ -i "pandas.Timestamp.day_name SA01" \
+ -i "pandas.Timestamp.day_of_week SA01" \
+ -i "pandas.Timestamp.day_of_year SA01" \
+ -i "pandas.Timestamp.dayofweek SA01" \
+ -i "pandas.Timestamp.dayofyear SA01" \
+ -i "pandas.Timestamp.days_in_month SA01" \
+ -i "pandas.Timestamp.daysinmonth SA01" \
+ -i "pandas.Timestamp.dst SA01" \
+ -i "pandas.Timestamp.floor SA01" \
+ -i "pandas.Timestamp.fold GL08" \
+ -i "pandas.Timestamp.fromordinal SA01" \
+ -i "pandas.Timestamp.fromtimestamp PR01,SA01" \
+ -i "pandas.Timestamp.hour GL08" \
+ -i "pandas.Timestamp.is_leap_year SA01" \
+ -i "pandas.Timestamp.isocalendar SA01" \
+ -i "pandas.Timestamp.isoformat SA01" \
+ -i "pandas.Timestamp.isoweekday SA01" \
+ -i "pandas.Timestamp.max PR02,PR07,SA01" \
+ -i "pandas.Timestamp.microsecond GL08" \
+ -i "pandas.Timestamp.min PR02,PR07,SA01" \
+ -i "pandas.Timestamp.minute GL08" \
+ -i "pandas.Timestamp.month GL08" \
+ -i "pandas.Timestamp.month_name SA01" \
+ -i "pandas.Timestamp.nanosecond GL08" \
+ -i "pandas.Timestamp.normalize SA01" \
+ -i "pandas.Timestamp.now SA01" \
+ -i "pandas.Timestamp.quarter SA01" \
+ -i "pandas.Timestamp.replace PR07,SA01" \
+ -i "pandas.Timestamp.resolution PR02,PR07,SA01" \
+ -i "pandas.Timestamp.round SA01" \
+ -i "pandas.Timestamp.second GL08" \
+ -i "pandas.Timestamp.strftime SA01" \
+ -i "pandas.Timestamp.strptime PR01,SA01" \
+ -i "pandas.Timestamp.time SA01" \
+ -i "pandas.Timestamp.timestamp SA01" \
+ -i "pandas.Timestamp.timetuple SA01" \
+ -i "pandas.Timestamp.timetz SA01" \
+ -i "pandas.Timestamp.to_datetime64 SA01" \
+ -i "pandas.Timestamp.to_julian_date SA01" \
+ -i "pandas.Timestamp.to_numpy PR01" \
+ -i "pandas.Timestamp.to_period PR01,SA01" \
+ -i "pandas.Timestamp.to_pydatetime PR01,SA01" \
+ -i "pandas.Timestamp.today SA01" \
+ -i "pandas.Timestamp.toordinal SA01" \
+ -i "pandas.Timestamp.tz SA01" \
+ -i "pandas.Timestamp.tz_convert SA01" \
+ -i "pandas.Timestamp.tz_localize SA01" \
+ -i "pandas.Timestamp.tzinfo GL08" \
+ -i "pandas.Timestamp.tzname SA01" \
+ -i "pandas.Timestamp.unit SA01" \
+ -i "pandas.Timestamp.utcfromtimestamp PR01,SA01" \
+ -i "pandas.Timestamp.utcnow SA01" \
+ -i "pandas.Timestamp.utcoffset SA01" \
+ -i "pandas.Timestamp.utctimetuple SA01" \
+ -i "pandas.Timestamp.value GL08" \
+ -i "pandas.Timestamp.week SA01" \
+ -i "pandas.Timestamp.weekday SA01" \
+ -i "pandas.Timestamp.weekofyear SA01" \
+ -i "pandas.Timestamp.year GL08" \
+ -i "pandas.UInt16Dtype SA01" \
+ -i "pandas.UInt32Dtype SA01" \
+ -i "pandas.UInt64Dtype SA01" \
+ -i "pandas.UInt8Dtype SA01" \
+ -i "pandas.api.extensions.ExtensionArray SA01" \
+ -i "pandas.api.extensions.ExtensionArray._accumulate RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray._concat_same_type PR07,SA01" \
+ -i "pandas.api.extensions.ExtensionArray._formatter SA01" \
+ -i "pandas.api.extensions.ExtensionArray._from_sequence SA01" \
+ -i "pandas.api.extensions.ExtensionArray._from_sequence_of_strings SA01" \
+ -i "pandas.api.extensions.ExtensionArray._hash_pandas_object RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray._pad_or_backfill PR01,RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray._reduce RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray._values_for_factorize SA01" \
+ -i "pandas.api.extensions.ExtensionArray.astype SA01" \
+ -i "pandas.api.extensions.ExtensionArray.copy RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray.dropna RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray.dtype SA01" \
+ -i "pandas.api.extensions.ExtensionArray.duplicated RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray.equals SA01" \
+ -i "pandas.api.extensions.ExtensionArray.fillna SA01" \
+ -i "pandas.api.extensions.ExtensionArray.insert PR07,RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray.interpolate PR01,SA01" \
+ -i "pandas.api.extensions.ExtensionArray.isin PR07,RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray.isna SA01" \
+ -i "pandas.api.extensions.ExtensionArray.nbytes SA01" \
+ -i "pandas.api.extensions.ExtensionArray.ndim SA01" \
+ -i "pandas.api.extensions.ExtensionArray.ravel RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray.shape SA01" \
+ -i "pandas.api.extensions.ExtensionArray.shift SA01" \
+ -i "pandas.api.extensions.ExtensionArray.take RT03" \
+ -i "pandas.api.extensions.ExtensionArray.tolist RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray.unique RT03,SA01" \
+ -i "pandas.api.extensions.ExtensionArray.view SA01" \
+ -i "pandas.api.extensions.register_extension_dtype SA01" \
+ -i "pandas.api.indexers.BaseIndexer PR01,SA01" \
+ -i "pandas.api.indexers.FixedForwardWindowIndexer PR01,SA01" \
+ -i "pandas.api.indexers.VariableOffsetWindowIndexer PR01,SA01" \
+ -i "pandas.api.interchange.from_dataframe RT03,SA01" \
+ -i "pandas.api.types.infer_dtype PR07,SA01" \
+ -i "pandas.api.types.is_any_real_numeric_dtype SA01" \
+ -i "pandas.api.types.is_bool PR01,SA01" \
+ -i "pandas.api.types.is_bool_dtype SA01" \
+ -i "pandas.api.types.is_categorical_dtype SA01" \
+ -i "pandas.api.types.is_complex PR01,SA01" \
+ -i "pandas.api.types.is_complex_dtype SA01" \
+ -i "pandas.api.types.is_datetime64_any_dtype SA01" \
+ -i "pandas.api.types.is_datetime64_dtype SA01" \
+ -i "pandas.api.types.is_datetime64_ns_dtype SA01" \
+ -i "pandas.api.types.is_datetime64tz_dtype SA01" \
+ -i "pandas.api.types.is_dict_like PR07,SA01" \
+ -i "pandas.api.types.is_extension_array_dtype SA01" \
+ -i "pandas.api.types.is_file_like PR07,SA01" \
+ -i "pandas.api.types.is_float PR01,SA01" \
+ -i "pandas.api.types.is_float_dtype SA01" \
+ -i "pandas.api.types.is_hashable PR01,RT03,SA01" \
+ -i "pandas.api.types.is_int64_dtype SA01" \
+ -i "pandas.api.types.is_integer PR01,SA01" \
+ -i "pandas.api.types.is_integer_dtype SA01" \
+ -i "pandas.api.types.is_interval_dtype SA01" \
+ -i "pandas.api.types.is_iterator PR07,SA01" \
+ -i "pandas.api.types.is_list_like SA01" \
+ -i "pandas.api.types.is_named_tuple PR07,SA01" \
+ -i "pandas.api.types.is_numeric_dtype SA01" \
+ -i "pandas.api.types.is_object_dtype SA01" \
+ -i "pandas.api.types.is_period_dtype SA01" \
+ -i "pandas.api.types.is_re PR07,SA01" \
+ -i "pandas.api.types.is_re_compilable PR07,SA01" \
+ -i "pandas.api.types.is_scalar SA01" \
+ -i "pandas.api.types.is_signed_integer_dtype SA01" \
+ -i "pandas.api.types.is_sparse SA01" \
+ -i "pandas.api.types.is_string_dtype SA01" \
+ -i "pandas.api.types.is_timedelta64_dtype SA01" \
+ -i "pandas.api.types.is_timedelta64_ns_dtype SA01" \
+ -i "pandas.api.types.is_unsigned_integer_dtype SA01" \
+ -i "pandas.api.types.pandas_dtype PR07,RT03,SA01" \
+ -i "pandas.api.types.union_categoricals RT03,SA01" \
+ -i "pandas.arrays.ArrowExtensionArray PR07,SA01" \
+ -i "pandas.arrays.BooleanArray SA01" \
+ -i "pandas.arrays.DatetimeArray SA01" \
+ -i "pandas.arrays.FloatingArray SA01" \
+ -i "pandas.arrays.IntegerArray SA01" \
+ -i "pandas.arrays.IntervalArray.closed SA01" \
+ -i "pandas.arrays.IntervalArray.contains RT03" \
+ -i "pandas.arrays.IntervalArray.is_non_overlapping_monotonic SA01" \
+ -i "pandas.arrays.IntervalArray.left SA01" \
+ -i "pandas.arrays.IntervalArray.length SA01" \
+ -i "pandas.arrays.IntervalArray.mid SA01" \
+ -i "pandas.arrays.IntervalArray.right SA01" \
+ -i "pandas.arrays.IntervalArray.set_closed RT03,SA01" \
+ -i "pandas.arrays.IntervalArray.to_tuples RT03,SA01" \
+ -i "pandas.arrays.NumpyExtensionArray SA01" \
+ -i "pandas.arrays.SparseArray PR07,SA01" \
+ -i "pandas.arrays.TimedeltaArray PR07,SA01" \
+ -i "pandas.bdate_range RT03,SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.__iter__ RT03,SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.agg RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.aggregate RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.apply RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.boxplot PR07,RT03,SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.cummax RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.cummin RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.cumprod RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.cumsum RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.filter RT03,SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.get_group RT03,SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.groups SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.hist RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.indices SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.max SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.mean RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.median SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.min SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.nth PR02" \
+ -i "pandas.core.groupby.DataFrameGroupBy.nunique RT03,SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.ohlc SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.plot PR02,SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.prod SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.rank RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.resample RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.sem SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.skew RT03" \
+ -i "pandas.core.groupby.DataFrameGroupBy.sum SA01" \
+ -i "pandas.core.groupby.DataFrameGroupBy.transform RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.__iter__ RT03,SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.agg RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.aggregate RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.apply RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.cummax RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.cummin RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.cumprod RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.cumsum RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.filter PR01,RT03,SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.get_group RT03,SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.groups SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.indices SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.is_monotonic_decreasing SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.is_monotonic_increasing SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.max SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.mean RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.median SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.min SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.nth PR02" \
+ -i "pandas.core.groupby.SeriesGroupBy.ohlc SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.plot PR02,SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.prod SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.rank RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.resample RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.sem SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.skew RT03" \
+ -i "pandas.core.groupby.SeriesGroupBy.sum SA01" \
+ -i "pandas.core.groupby.SeriesGroupBy.transform RT03" \
+ -i "pandas.core.resample.Resampler.__iter__ RT03,SA01" \
+ -i "pandas.core.resample.Resampler.ffill RT03" \
+ -i "pandas.core.resample.Resampler.get_group RT03,SA01" \
+ -i "pandas.core.resample.Resampler.groups SA01" \
+ -i "pandas.core.resample.Resampler.indices SA01" \
+ -i "pandas.core.resample.Resampler.max PR01,RT03,SA01" \
+ -i "pandas.core.resample.Resampler.mean SA01" \
+ -i "pandas.core.resample.Resampler.median SA01" \
+ -i "pandas.core.resample.Resampler.min PR01,RT03,SA01" \
+ -i "pandas.core.resample.Resampler.ohlc SA01" \
+ -i "pandas.core.resample.Resampler.prod SA01" \
+ -i "pandas.core.resample.Resampler.quantile PR01,PR07" \
+ -i "pandas.core.resample.Resampler.sem SA01" \
+ -i "pandas.core.resample.Resampler.std SA01" \
+ -i "pandas.core.resample.Resampler.sum SA01" \
+ -i "pandas.core.resample.Resampler.transform PR01,RT03,SA01" \
+ -i "pandas.core.resample.Resampler.var SA01" \
+ -i "pandas.core.window.expanding.Expanding.corr PR01" \
+ -i "pandas.core.window.expanding.Expanding.count PR01" \
+ -i "pandas.core.window.rolling.Rolling.max PR01" \
+ -i "pandas.core.window.rolling.Window.std PR01" \
+ -i "pandas.core.window.rolling.Window.var PR01" \
+ -i "pandas.date_range RT03" \
+ -i "pandas.describe_option SA01" \
+ -i "pandas.errors.AbstractMethodError PR01,SA01" \
+ -i "pandas.errors.AttributeConflictWarning SA01" \
+ -i "pandas.errors.CSSWarning SA01" \
+ -i "pandas.errors.CategoricalConversionWarning SA01" \
+ -i "pandas.errors.ChainedAssignmentError SA01" \
+ -i "pandas.errors.ClosedFileError SA01" \
+ -i "pandas.errors.DataError SA01" \
+ -i "pandas.errors.DuplicateLabelError SA01" \
+ -i "pandas.errors.EmptyDataError SA01" \
+ -i "pandas.errors.IntCastingNaNError SA01" \
+ -i "pandas.errors.InvalidIndexError SA01" \
+ -i "pandas.errors.InvalidVersion SA01" \
+ -i "pandas.errors.MergeError SA01" \
+ -i "pandas.errors.NullFrequencyError SA01" \
+ -i "pandas.errors.NumExprClobberingError SA01" \
+ -i "pandas.errors.NumbaUtilError SA01" \
+ -i "pandas.errors.OptionError SA01" \
+ -i "pandas.errors.OutOfBoundsDatetime SA01" \
+ -i "pandas.errors.OutOfBoundsTimedelta SA01" \
+ -i "pandas.errors.PerformanceWarning SA01" \
+ -i "pandas.errors.PossibleDataLossError SA01" \
+ -i "pandas.errors.PossiblePrecisionLoss SA01" \
+ -i "pandas.errors.SpecificationError SA01" \
+ -i "pandas.errors.UndefinedVariableError PR01,SA01" \
+ -i "pandas.errors.UnsortedIndexError SA01" \
+ -i "pandas.errors.UnsupportedFunctionCall SA01" \
+ -i "pandas.errors.ValueLabelTypeMismatch SA01" \
+ -i "pandas.get_option SA01" \
+ -i "pandas.infer_freq SA01" \
+ -i "pandas.interval_range RT03" \
+ -i "pandas.io.formats.style.Styler.apply RT03" \
+ -i "pandas.io.formats.style.Styler.apply_index RT03" \
+ -i "pandas.io.formats.style.Styler.background_gradient RT03" \
+ -i "pandas.io.formats.style.Styler.bar RT03,SA01" \
+ -i "pandas.io.formats.style.Styler.clear SA01" \
+ -i "pandas.io.formats.style.Styler.concat RT03,SA01" \
+ -i "pandas.io.formats.style.Styler.export RT03" \
+ -i "pandas.io.formats.style.Styler.from_custom_template SA01" \
+ -i "pandas.io.formats.style.Styler.hide RT03,SA01" \
+ -i "pandas.io.formats.style.Styler.highlight_between RT03" \
+ -i "pandas.io.formats.style.Styler.highlight_max RT03" \
+ -i "pandas.io.formats.style.Styler.highlight_min RT03" \
+ -i "pandas.io.formats.style.Styler.highlight_null RT03" \
+ -i "pandas.io.formats.style.Styler.highlight_quantile RT03" \
+ -i "pandas.io.formats.style.Styler.map RT03" \
+ -i "pandas.io.formats.style.Styler.map_index RT03" \
+ -i "pandas.io.formats.style.Styler.set_caption RT03,SA01" \
+ -i "pandas.io.formats.style.Styler.set_properties RT03,SA01" \
+ -i "pandas.io.formats.style.Styler.set_sticky RT03,SA01" \
+ -i "pandas.io.formats.style.Styler.set_table_attributes PR07,RT03" \
+ -i "pandas.io.formats.style.Styler.set_table_styles RT03" \
+ -i "pandas.io.formats.style.Styler.set_td_classes RT03" \
+ -i "pandas.io.formats.style.Styler.set_tooltips RT03,SA01" \
+ -i "pandas.io.formats.style.Styler.set_uuid PR07,RT03,SA01" \
+ -i "pandas.io.formats.style.Styler.text_gradient RT03" \
+ -i "pandas.io.formats.style.Styler.to_excel PR01" \
+ -i "pandas.io.formats.style.Styler.to_string SA01" \
+ -i "pandas.io.formats.style.Styler.use RT03" \
+ -i "pandas.io.json.build_table_schema PR07,RT03,SA01" \
+ -i "pandas.io.stata.StataReader.data_label SA01" \
+ -i "pandas.io.stata.StataReader.value_labels RT03,SA01" \
+ -i "pandas.io.stata.StataReader.variable_labels RT03,SA01" \
+ -i "pandas.io.stata.StataWriter.write_file SA01" \
+ -i "pandas.json_normalize RT03,SA01" \
+ -i "pandas.merge PR07" \
+ -i "pandas.merge_asof PR07,RT03" \
+ -i "pandas.merge_ordered PR07" \
+ -i "pandas.option_context SA01" \
+ -i "pandas.period_range RT03,SA01" \
+ -i "pandas.pivot PR07" \
+ -i "pandas.pivot_table PR07" \
+ -i "pandas.plotting.andrews_curves RT03,SA01" \
+ -i "pandas.plotting.autocorrelation_plot RT03,SA01" \
+ -i "pandas.plotting.lag_plot RT03,SA01" \
+ -i "pandas.plotting.parallel_coordinates PR07,RT03,SA01" \
+ -i "pandas.plotting.plot_params SA01" \
+ -i "pandas.plotting.scatter_matrix PR07,SA01" \
+ -i "pandas.plotting.table PR07,RT03,SA01" \
+ -i "pandas.qcut PR07,SA01" \
+ -i "pandas.read_feather SA01" \
+ -i "pandas.read_orc SA01" \
+ -i "pandas.read_sas SA01" \
+ -i "pandas.read_spss SA01" \
+ -i "pandas.reset_option SA01" \
+ -i "pandas.set_eng_float_format RT03,SA01" \
+ -i "pandas.set_option SA01" \
+ -i "pandas.show_versions SA01" \
+ -i "pandas.test SA01" \
+ -i "pandas.testing.assert_extension_array_equal SA01" \
+ -i "pandas.testing.assert_index_equal PR07,SA01" \
+ -i "pandas.testing.assert_series_equal PR07,SA01" \
+ -i "pandas.timedelta_range SA01" \
+ -i "pandas.tseries.api.guess_datetime_format SA01" \
+ -i "pandas.tseries.offsets.BDay PR02,SA01" \
+ -i "pandas.tseries.offsets.BMonthBegin PR02" \
+ -i "pandas.tseries.offsets.BMonthEnd PR02" \
+ -i "pandas.tseries.offsets.BQuarterBegin PR02" \
+ -i "pandas.tseries.offsets.BQuarterBegin.copy SA01" \
+ -i "pandas.tseries.offsets.BQuarterBegin.freqstr SA01" \
+ -i "pandas.tseries.offsets.BQuarterBegin.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.BQuarterBegin.kwds SA01" \
+ -i "pandas.tseries.offsets.BQuarterBegin.n GL08" \
+ -i "pandas.tseries.offsets.BQuarterBegin.name SA01" \
+ -i "pandas.tseries.offsets.BQuarterBegin.nanos GL08" \
+ -i "pandas.tseries.offsets.BQuarterBegin.normalize GL08" \
+ -i "pandas.tseries.offsets.BQuarterBegin.rule_code GL08" \
+ -i "pandas.tseries.offsets.BQuarterBegin.startingMonth GL08" \
+ -i "pandas.tseries.offsets.BQuarterEnd PR02" \
+ -i "pandas.tseries.offsets.BQuarterEnd.copy SA01" \
+ -i "pandas.tseries.offsets.BQuarterEnd.freqstr SA01" \
+ -i "pandas.tseries.offsets.BQuarterEnd.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.BQuarterEnd.kwds SA01" \
+ -i "pandas.tseries.offsets.BQuarterEnd.n GL08" \
+ -i "pandas.tseries.offsets.BQuarterEnd.name SA01" \
+ -i "pandas.tseries.offsets.BQuarterEnd.nanos GL08" \
+ -i "pandas.tseries.offsets.BQuarterEnd.normalize GL08" \
+ -i "pandas.tseries.offsets.BQuarterEnd.rule_code GL08" \
+ -i "pandas.tseries.offsets.BQuarterEnd.startingMonth GL08" \
+ -i "pandas.tseries.offsets.BYearBegin PR02" \
+ -i "pandas.tseries.offsets.BYearBegin.copy SA01" \
+ -i "pandas.tseries.offsets.BYearBegin.freqstr SA01" \
+ -i "pandas.tseries.offsets.BYearBegin.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.BYearBegin.kwds SA01" \
+ -i "pandas.tseries.offsets.BYearBegin.month GL08" \
+ -i "pandas.tseries.offsets.BYearBegin.n GL08" \
+ -i "pandas.tseries.offsets.BYearBegin.name SA01" \
+ -i "pandas.tseries.offsets.BYearBegin.nanos GL08" \
+ -i "pandas.tseries.offsets.BYearBegin.normalize GL08" \
+ -i "pandas.tseries.offsets.BYearBegin.rule_code GL08" \
+ -i "pandas.tseries.offsets.BYearEnd PR02" \
+ -i "pandas.tseries.offsets.BYearEnd.copy SA01" \
+ -i "pandas.tseries.offsets.BYearEnd.freqstr SA01" \
+ -i "pandas.tseries.offsets.BYearEnd.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.BYearEnd.kwds SA01" \
+ -i "pandas.tseries.offsets.BYearEnd.month GL08" \
+ -i "pandas.tseries.offsets.BYearEnd.n GL08" \
+ -i "pandas.tseries.offsets.BYearEnd.name SA01" \
+ -i "pandas.tseries.offsets.BYearEnd.nanos GL08" \
+ -i "pandas.tseries.offsets.BYearEnd.normalize GL08" \
+ -i "pandas.tseries.offsets.BYearEnd.rule_code GL08" \
+ -i "pandas.tseries.offsets.BusinessDay PR02,SA01" \
+ -i "pandas.tseries.offsets.BusinessDay.calendar GL08" \
+ -i "pandas.tseries.offsets.BusinessDay.copy SA01" \
+ -i "pandas.tseries.offsets.BusinessDay.freqstr SA01" \
+ -i "pandas.tseries.offsets.BusinessDay.holidays GL08" \
+ -i "pandas.tseries.offsets.BusinessDay.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.BusinessDay.kwds SA01" \
+ -i "pandas.tseries.offsets.BusinessDay.n GL08" \
+ -i "pandas.tseries.offsets.BusinessDay.name SA01" \
+ -i "pandas.tseries.offsets.BusinessDay.nanos GL08" \
+ -i "pandas.tseries.offsets.BusinessDay.normalize GL08" \
+ -i "pandas.tseries.offsets.BusinessDay.rule_code GL08" \
+ -i "pandas.tseries.offsets.BusinessDay.weekmask GL08" \
+ -i "pandas.tseries.offsets.BusinessHour PR02,SA01" \
+ -i "pandas.tseries.offsets.BusinessHour.calendar GL08" \
+ -i "pandas.tseries.offsets.BusinessHour.copy SA01" \
+ -i "pandas.tseries.offsets.BusinessHour.end GL08" \
+ -i "pandas.tseries.offsets.BusinessHour.freqstr SA01" \
+ -i "pandas.tseries.offsets.BusinessHour.holidays GL08" \
+ -i "pandas.tseries.offsets.BusinessHour.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.BusinessHour.kwds SA01" \
+ -i "pandas.tseries.offsets.BusinessHour.n GL08" \
+ -i "pandas.tseries.offsets.BusinessHour.name SA01" \
+ -i "pandas.tseries.offsets.BusinessHour.nanos GL08" \
+ -i "pandas.tseries.offsets.BusinessHour.normalize GL08" \
+ -i "pandas.tseries.offsets.BusinessHour.rule_code GL08" \
+ -i "pandas.tseries.offsets.BusinessHour.start GL08" \
+ -i "pandas.tseries.offsets.BusinessHour.weekmask GL08" \
+ -i "pandas.tseries.offsets.BusinessMonthBegin PR02" \
+ -i "pandas.tseries.offsets.BusinessMonthBegin.copy SA01" \
+ -i "pandas.tseries.offsets.BusinessMonthBegin.freqstr SA01" \
+ -i "pandas.tseries.offsets.BusinessMonthBegin.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.BusinessMonthBegin.kwds SA01" \
+ -i "pandas.tseries.offsets.BusinessMonthBegin.n GL08" \
+ -i "pandas.tseries.offsets.BusinessMonthBegin.name SA01" \
+ -i "pandas.tseries.offsets.BusinessMonthBegin.nanos GL08" \
+ -i "pandas.tseries.offsets.BusinessMonthBegin.normalize GL08" \
+ -i "pandas.tseries.offsets.BusinessMonthBegin.rule_code GL08" \
+ -i "pandas.tseries.offsets.BusinessMonthEnd PR02" \
+ -i "pandas.tseries.offsets.BusinessMonthEnd.copy SA01" \
+ -i "pandas.tseries.offsets.BusinessMonthEnd.freqstr SA01" \
+ -i "pandas.tseries.offsets.BusinessMonthEnd.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.BusinessMonthEnd.kwds SA01" \
+ -i "pandas.tseries.offsets.BusinessMonthEnd.n GL08" \
+ -i "pandas.tseries.offsets.BusinessMonthEnd.name SA01" \
+ -i "pandas.tseries.offsets.BusinessMonthEnd.nanos GL08" \
+ -i "pandas.tseries.offsets.BusinessMonthEnd.normalize GL08" \
+ -i "pandas.tseries.offsets.BusinessMonthEnd.rule_code GL08" \
+ -i "pandas.tseries.offsets.CBMonthBegin PR02" \
+ -i "pandas.tseries.offsets.CBMonthEnd PR02" \
+ -i "pandas.tseries.offsets.CDay PR02,SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessDay PR02,SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.calendar GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.copy SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.freqstr SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.holidays GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.kwds SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.n GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.name SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.nanos GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.normalize GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.rule_code GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessDay.weekmask GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessHour PR02,SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.calendar GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.copy SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.end GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.freqstr SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.holidays GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.kwds SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.n GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.name SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.nanos GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.normalize GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.rule_code GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.start GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessHour.weekmask GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin PR02" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.calendar GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.copy SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.freqstr SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.holidays GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.is_on_offset SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.kwds SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.m_offset GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.n GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.name SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.nanos GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.normalize GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.rule_code GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthBegin.weekmask GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd PR02" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.calendar GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.copy SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.freqstr SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.holidays GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.is_on_offset SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.kwds SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.m_offset GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.n GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.name SA01" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.nanos GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.normalize GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.rule_code GL08" \
+ -i "pandas.tseries.offsets.CustomBusinessMonthEnd.weekmask GL08" \
+ -i "pandas.tseries.offsets.DateOffset PR02" \
+ -i "pandas.tseries.offsets.DateOffset.copy SA01" \
+ -i "pandas.tseries.offsets.DateOffset.freqstr SA01" \
+ -i "pandas.tseries.offsets.DateOffset.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.DateOffset.kwds SA01" \
+ -i "pandas.tseries.offsets.DateOffset.n GL08" \
+ -i "pandas.tseries.offsets.DateOffset.name SA01" \
+ -i "pandas.tseries.offsets.DateOffset.nanos GL08" \
+ -i "pandas.tseries.offsets.DateOffset.normalize GL08" \
+ -i "pandas.tseries.offsets.DateOffset.rule_code GL08" \
+ -i "pandas.tseries.offsets.Day PR02" \
+ -i "pandas.tseries.offsets.Day.copy SA01" \
+ -i "pandas.tseries.offsets.Day.freqstr SA01" \
+ -i "pandas.tseries.offsets.Day.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.Day.kwds SA01" \
+ -i "pandas.tseries.offsets.Day.n GL08" \
+ -i "pandas.tseries.offsets.Day.name SA01" \
+ -i "pandas.tseries.offsets.Day.nanos SA01" \
+ -i "pandas.tseries.offsets.Day.normalize GL08" \
+ -i "pandas.tseries.offsets.Day.rule_code GL08" \
+ -i "pandas.tseries.offsets.Easter PR02" \
+ -i "pandas.tseries.offsets.Easter.copy SA01" \
+ -i "pandas.tseries.offsets.Easter.freqstr SA01" \
+ -i "pandas.tseries.offsets.Easter.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.Easter.kwds SA01" \
+ -i "pandas.tseries.offsets.Easter.n GL08" \
+ -i "pandas.tseries.offsets.Easter.name SA01" \
+ -i "pandas.tseries.offsets.Easter.nanos GL08" \
+ -i "pandas.tseries.offsets.Easter.normalize GL08" \
+ -i "pandas.tseries.offsets.Easter.rule_code GL08" \
+ -i "pandas.tseries.offsets.FY5253 PR02" \
+ -i "pandas.tseries.offsets.FY5253.copy SA01" \
+ -i "pandas.tseries.offsets.FY5253.freqstr SA01" \
+ -i "pandas.tseries.offsets.FY5253.get_rule_code_suffix GL08" \
+ -i "pandas.tseries.offsets.FY5253.get_year_end GL08" \
+ -i "pandas.tseries.offsets.FY5253.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.FY5253.kwds SA01" \
+ -i "pandas.tseries.offsets.FY5253.n GL08" \
+ -i "pandas.tseries.offsets.FY5253.name SA01" \
+ -i "pandas.tseries.offsets.FY5253.nanos GL08" \
+ -i "pandas.tseries.offsets.FY5253.normalize GL08" \
+ -i "pandas.tseries.offsets.FY5253.rule_code GL08" \
+ -i "pandas.tseries.offsets.FY5253.startingMonth GL08" \
+ -i "pandas.tseries.offsets.FY5253.variation GL08" \
+ -i "pandas.tseries.offsets.FY5253.weekday GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter PR02" \
+ -i "pandas.tseries.offsets.FY5253Quarter.copy SA01" \
+ -i "pandas.tseries.offsets.FY5253Quarter.freqstr SA01" \
+ -i "pandas.tseries.offsets.FY5253Quarter.get_rule_code_suffix GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.get_weeks GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.kwds SA01" \
+ -i "pandas.tseries.offsets.FY5253Quarter.n GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.name SA01" \
+ -i "pandas.tseries.offsets.FY5253Quarter.nanos GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.normalize GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.qtr_with_extra_week GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.rule_code GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.startingMonth GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.variation GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.weekday GL08" \
+ -i "pandas.tseries.offsets.FY5253Quarter.year_has_extra_week GL08" \
+ -i "pandas.tseries.offsets.Hour PR02" \
+ -i "pandas.tseries.offsets.Hour.copy SA01" \
+ -i "pandas.tseries.offsets.Hour.freqstr SA01" \
+ -i "pandas.tseries.offsets.Hour.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.Hour.kwds SA01" \
+ -i "pandas.tseries.offsets.Hour.n GL08" \
+ -i "pandas.tseries.offsets.Hour.name SA01" \
+ -i "pandas.tseries.offsets.Hour.nanos SA01" \
+ -i "pandas.tseries.offsets.Hour.normalize GL08" \
+ -i "pandas.tseries.offsets.Hour.rule_code GL08" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth PR02,SA01" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.copy SA01" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.freqstr SA01" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.kwds SA01" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.n GL08" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.name SA01" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.nanos GL08" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.normalize GL08" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.rule_code GL08" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.week GL08" \
+ -i "pandas.tseries.offsets.LastWeekOfMonth.weekday GL08" \
+ -i "pandas.tseries.offsets.Micro PR02" \
+ -i "pandas.tseries.offsets.Micro.copy SA01" \
+ -i "pandas.tseries.offsets.Micro.freqstr SA01" \
+ -i "pandas.tseries.offsets.Micro.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.Micro.kwds SA01" \
+ -i "pandas.tseries.offsets.Micro.n GL08" \
+ -i "pandas.tseries.offsets.Micro.name SA01" \
+ -i "pandas.tseries.offsets.Micro.nanos SA01" \
+ -i "pandas.tseries.offsets.Micro.normalize GL08" \
+ -i "pandas.tseries.offsets.Micro.rule_code GL08" \
+ -i "pandas.tseries.offsets.Milli PR02" \
+ -i "pandas.tseries.offsets.Milli.copy SA01" \
+ -i "pandas.tseries.offsets.Milli.freqstr SA01" \
+ -i "pandas.tseries.offsets.Milli.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.Milli.kwds SA01" \
+ -i "pandas.tseries.offsets.Milli.n GL08" \
+ -i "pandas.tseries.offsets.Milli.name SA01" \
+ -i "pandas.tseries.offsets.Milli.nanos SA01" \
+ -i "pandas.tseries.offsets.Milli.normalize GL08" \
+ -i "pandas.tseries.offsets.Milli.rule_code GL08" \
+ -i "pandas.tseries.offsets.Minute PR02" \
+ -i "pandas.tseries.offsets.Minute.copy SA01" \
+ -i "pandas.tseries.offsets.Minute.freqstr SA01" \
+ -i "pandas.tseries.offsets.Minute.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.Minute.kwds SA01" \
+ -i "pandas.tseries.offsets.Minute.n GL08" \
+ -i "pandas.tseries.offsets.Minute.name SA01" \
+ -i "pandas.tseries.offsets.Minute.nanos SA01" \
+ -i "pandas.tseries.offsets.Minute.normalize GL08" \
+ -i "pandas.tseries.offsets.Minute.rule_code GL08" \
+ -i "pandas.tseries.offsets.MonthBegin PR02" \
+ -i "pandas.tseries.offsets.MonthBegin.copy SA01" \
+ -i "pandas.tseries.offsets.MonthBegin.freqstr SA01" \
+ -i "pandas.tseries.offsets.MonthBegin.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.MonthBegin.kwds SA01" \
+ -i "pandas.tseries.offsets.MonthBegin.n GL08" \
+ -i "pandas.tseries.offsets.MonthBegin.name SA01" \
+ -i "pandas.tseries.offsets.MonthBegin.nanos GL08" \
+ -i "pandas.tseries.offsets.MonthBegin.normalize GL08" \
+ -i "pandas.tseries.offsets.MonthBegin.rule_code GL08" \
+ -i "pandas.tseries.offsets.MonthEnd PR02" \
+ -i "pandas.tseries.offsets.MonthEnd.copy SA01" \
+ -i "pandas.tseries.offsets.MonthEnd.freqstr SA01" \
+ -i "pandas.tseries.offsets.MonthEnd.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.MonthEnd.kwds SA01" \
+ -i "pandas.tseries.offsets.MonthEnd.n GL08" \
+ -i "pandas.tseries.offsets.MonthEnd.name SA01" \
+ -i "pandas.tseries.offsets.MonthEnd.nanos GL08" \
+ -i "pandas.tseries.offsets.MonthEnd.normalize GL08" \
+ -i "pandas.tseries.offsets.MonthEnd.rule_code GL08" \
+ -i "pandas.tseries.offsets.Nano PR02" \
+ -i "pandas.tseries.offsets.Nano.copy SA01" \
+ -i "pandas.tseries.offsets.Nano.freqstr SA01" \
+ -i "pandas.tseries.offsets.Nano.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.Nano.kwds SA01" \
+ -i "pandas.tseries.offsets.Nano.n GL08" \
+ -i "pandas.tseries.offsets.Nano.name SA01" \
+ -i "pandas.tseries.offsets.Nano.nanos SA01" \
+ -i "pandas.tseries.offsets.Nano.normalize GL08" \
+ -i "pandas.tseries.offsets.Nano.rule_code GL08" \
+ -i "pandas.tseries.offsets.QuarterBegin PR02" \
+ -i "pandas.tseries.offsets.QuarterBegin.copy SA01" \
+ -i "pandas.tseries.offsets.QuarterBegin.freqstr SA01" \
+ -i "pandas.tseries.offsets.QuarterBegin.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.QuarterBegin.kwds SA01" \
+ -i "pandas.tseries.offsets.QuarterBegin.n GL08" \
+ -i "pandas.tseries.offsets.QuarterBegin.name SA01" \
+ -i "pandas.tseries.offsets.QuarterBegin.nanos GL08" \
+ -i "pandas.tseries.offsets.QuarterBegin.normalize GL08" \
+ -i "pandas.tseries.offsets.QuarterBegin.rule_code GL08" \
+ -i "pandas.tseries.offsets.QuarterBegin.startingMonth GL08" \
+ -i "pandas.tseries.offsets.QuarterEnd PR02" \
+ -i "pandas.tseries.offsets.QuarterEnd.copy SA01" \
+ -i "pandas.tseries.offsets.QuarterEnd.freqstr SA01" \
+ -i "pandas.tseries.offsets.QuarterEnd.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.QuarterEnd.kwds SA01" \
+ -i "pandas.tseries.offsets.QuarterEnd.n GL08" \
+ -i "pandas.tseries.offsets.QuarterEnd.name SA01" \
+ -i "pandas.tseries.offsets.QuarterEnd.nanos GL08" \
+ -i "pandas.tseries.offsets.QuarterEnd.normalize GL08" \
+ -i "pandas.tseries.offsets.QuarterEnd.rule_code GL08" \
+ -i "pandas.tseries.offsets.QuarterEnd.startingMonth GL08" \
+ -i "pandas.tseries.offsets.Second PR02" \
+ -i "pandas.tseries.offsets.Second.copy SA01" \
+ -i "pandas.tseries.offsets.Second.freqstr SA01" \
+ -i "pandas.tseries.offsets.Second.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.Second.kwds SA01" \
+ -i "pandas.tseries.offsets.Second.n GL08" \
+ -i "pandas.tseries.offsets.Second.name SA01" \
+ -i "pandas.tseries.offsets.Second.nanos SA01" \
+ -i "pandas.tseries.offsets.Second.normalize GL08" \
+ -i "pandas.tseries.offsets.Second.rule_code GL08" \
+ -i "pandas.tseries.offsets.SemiMonthBegin PR02,SA01" \
+ -i "pandas.tseries.offsets.SemiMonthBegin.copy SA01" \
+ -i "pandas.tseries.offsets.SemiMonthBegin.day_of_month GL08" \
+ -i "pandas.tseries.offsets.SemiMonthBegin.freqstr SA01" \
+ -i "pandas.tseries.offsets.SemiMonthBegin.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.SemiMonthBegin.kwds SA01" \
+ -i "pandas.tseries.offsets.SemiMonthBegin.n GL08" \
+ -i "pandas.tseries.offsets.SemiMonthBegin.name SA01" \
+ -i "pandas.tseries.offsets.SemiMonthBegin.nanos GL08" \
+ -i "pandas.tseries.offsets.SemiMonthBegin.normalize GL08" \
+ -i "pandas.tseries.offsets.SemiMonthBegin.rule_code GL08" \
+ -i "pandas.tseries.offsets.SemiMonthEnd PR02,SA01" \
+ -i "pandas.tseries.offsets.SemiMonthEnd.copy SA01" \
+ -i "pandas.tseries.offsets.SemiMonthEnd.day_of_month GL08" \
+ -i "pandas.tseries.offsets.SemiMonthEnd.freqstr SA01" \
+ -i "pandas.tseries.offsets.SemiMonthEnd.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.SemiMonthEnd.kwds SA01" \
+ -i "pandas.tseries.offsets.SemiMonthEnd.n GL08" \
+ -i "pandas.tseries.offsets.SemiMonthEnd.name SA01" \
+ -i "pandas.tseries.offsets.SemiMonthEnd.nanos GL08" \
+ -i "pandas.tseries.offsets.SemiMonthEnd.normalize GL08" \
+ -i "pandas.tseries.offsets.SemiMonthEnd.rule_code GL08" \
+ -i "pandas.tseries.offsets.Tick GL08" \
+ -i "pandas.tseries.offsets.Tick.copy SA01" \
+ -i "pandas.tseries.offsets.Tick.freqstr SA01" \
+ -i "pandas.tseries.offsets.Tick.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.Tick.kwds SA01" \
+ -i "pandas.tseries.offsets.Tick.n GL08" \
+ -i "pandas.tseries.offsets.Tick.name SA01" \
+ -i "pandas.tseries.offsets.Tick.nanos SA01" \
+ -i "pandas.tseries.offsets.Tick.normalize GL08" \
+ -i "pandas.tseries.offsets.Tick.rule_code GL08" \
+ -i "pandas.tseries.offsets.Week PR02" \
+ -i "pandas.tseries.offsets.Week.copy SA01" \
+ -i "pandas.tseries.offsets.Week.freqstr SA01" \
+ -i "pandas.tseries.offsets.Week.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.Week.kwds SA01" \
+ -i "pandas.tseries.offsets.Week.n GL08" \
+ -i "pandas.tseries.offsets.Week.name SA01" \
+ -i "pandas.tseries.offsets.Week.nanos GL08" \
+ -i "pandas.tseries.offsets.Week.normalize GL08" \
+ -i "pandas.tseries.offsets.Week.rule_code GL08" \
+ -i "pandas.tseries.offsets.Week.weekday GL08" \
+ -i "pandas.tseries.offsets.WeekOfMonth PR02,SA01" \
+ -i "pandas.tseries.offsets.WeekOfMonth.copy SA01" \
+ -i "pandas.tseries.offsets.WeekOfMonth.freqstr SA01" \
+ -i "pandas.tseries.offsets.WeekOfMonth.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.WeekOfMonth.kwds SA01" \
+ -i "pandas.tseries.offsets.WeekOfMonth.n GL08" \
+ -i "pandas.tseries.offsets.WeekOfMonth.name SA01" \
+ -i "pandas.tseries.offsets.WeekOfMonth.nanos GL08" \
+ -i "pandas.tseries.offsets.WeekOfMonth.normalize GL08" \
+ -i "pandas.tseries.offsets.WeekOfMonth.rule_code GL08" \
+ -i "pandas.tseries.offsets.WeekOfMonth.week GL08" \
+ -i "pandas.tseries.offsets.WeekOfMonth.weekday GL08" \
+ -i "pandas.tseries.offsets.YearBegin PR02" \
+ -i "pandas.tseries.offsets.YearBegin.copy SA01" \
+ -i "pandas.tseries.offsets.YearBegin.freqstr SA01" \
+ -i "pandas.tseries.offsets.YearBegin.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.YearBegin.kwds SA01" \
+ -i "pandas.tseries.offsets.YearBegin.month GL08" \
+ -i "pandas.tseries.offsets.YearBegin.n GL08" \
+ -i "pandas.tseries.offsets.YearBegin.name SA01" \
+ -i "pandas.tseries.offsets.YearBegin.nanos GL08" \
+ -i "pandas.tseries.offsets.YearBegin.normalize GL08" \
+ -i "pandas.tseries.offsets.YearBegin.rule_code GL08" \
+ -i "pandas.tseries.offsets.YearEnd PR02" \
+ -i "pandas.tseries.offsets.YearEnd.copy SA01" \
+ -i "pandas.tseries.offsets.YearEnd.freqstr SA01" \
+ -i "pandas.tseries.offsets.YearEnd.is_on_offset GL08" \
+ -i "pandas.tseries.offsets.YearEnd.kwds SA01" \
+ -i "pandas.tseries.offsets.YearEnd.month GL08" \
+ -i "pandas.tseries.offsets.YearEnd.n GL08" \
+ -i "pandas.tseries.offsets.YearEnd.name SA01" \
+ -i "pandas.tseries.offsets.YearEnd.nanos GL08" \
+ -i "pandas.tseries.offsets.YearEnd.normalize GL08" \
+ -i "pandas.tseries.offsets.YearEnd.rule_code GL08" \
+ -i "pandas.unique PR07" \
+ -i "pandas.util.hash_array PR07,SA01" \
+ -i "pandas.util.hash_pandas_object PR07,SA01" # There should be no backslash in the final line, please keep this comment in the last ignored function
- MSG='Partially validate docstrings (EX03)' ; echo $MSG
- $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX03 --ignore_functions \
- pandas.Series.dt.day_name \
- pandas.Series.str.len \
- pandas.Series.cat.set_categories \
- pandas.Series.plot.bar \
- pandas.Series.plot.hist \
- pandas.Series.plot.line \
- pandas.Series.to_sql \
- pandas.Series.to_latex \
- pandas.errors.CategoricalConversionWarning \
- pandas.errors.ChainedAssignmentError \
- pandas.errors.ClosedFileError \
- pandas.errors.DatabaseError \
- pandas.errors.IndexingError \
- pandas.errors.InvalidColumnName \
- pandas.errors.NumExprClobberingError \
- pandas.errors.PossibleDataLossError \
- pandas.errors.PossiblePrecisionLoss \
- pandas.errors.SettingWithCopyError \
- pandas.errors.SettingWithCopyWarning \
- pandas.errors.SpecificationError \
- pandas.errors.UndefinedVariableError \
- pandas.errors.ValueLabelTypeMismatch \
- pandas.Timestamp.ceil \
- pandas.Timestamp.floor \
- pandas.Timestamp.round \
- pandas.read_pickle \
- pandas.ExcelWriter \
- pandas.read_json \
- pandas.io.json.build_table_schema \
- pandas.DataFrame.to_latex \
- pandas.io.formats.style.Styler.to_latex \
- pandas.read_parquet \
- pandas.DataFrame.to_sql \
- pandas.read_stata \
- pandas.core.resample.Resampler.pipe \
- pandas.core.resample.Resampler.fillna \
- pandas.core.resample.Resampler.interpolate \
- pandas.plotting.scatter_matrix \
- pandas.pivot \
- pandas.merge_asof \
- pandas.wide_to_long \
- pandas.Index.rename \
- pandas.Index.droplevel \
- pandas.Index.isin \
- pandas.CategoricalIndex.set_categories \
- pandas.MultiIndex.names \
- pandas.MultiIndex.droplevel \
- pandas.IndexSlice \
- pandas.DatetimeIndex.month_name \
- pandas.DatetimeIndex.day_name \
- pandas.core.window.rolling.Rolling.corr \
- pandas.Grouper \
- pandas.core.groupby.SeriesGroupBy.apply \
- pandas.core.groupby.DataFrameGroupBy.apply \
- pandas.core.groupby.SeriesGroupBy.transform \
- pandas.core.groupby.SeriesGroupBy.pipe \
- pandas.core.groupby.DataFrameGroupBy.pipe \
- pandas.core.groupby.DataFrameGroupBy.describe \
- pandas.core.groupby.DataFrameGroupBy.idxmax \
- pandas.core.groupby.DataFrameGroupBy.idxmin \
- pandas.core.groupby.DataFrameGroupBy.value_counts \
- pandas.core.groupby.SeriesGroupBy.describe \
- pandas.core.groupby.DataFrameGroupBy.boxplot \
- pandas.core.groupby.DataFrameGroupBy.hist \
- pandas.io.formats.style.Styler.map \
- pandas.io.formats.style.Styler.apply_index \
- pandas.io.formats.style.Styler.map_index \
- pandas.io.formats.style.Styler.format \
- pandas.io.formats.style.Styler.format_index \
- pandas.io.formats.style.Styler.relabel_index \
- pandas.io.formats.style.Styler.hide \
- pandas.io.formats.style.Styler.set_td_classes \
- pandas.io.formats.style.Styler.set_tooltips \
- pandas.io.formats.style.Styler.set_uuid \
- pandas.io.formats.style.Styler.pipe \
- pandas.io.formats.style.Styler.highlight_between \
- pandas.io.formats.style.Styler.highlight_quantile \
- pandas.io.formats.style.Styler.background_gradient \
- pandas.io.formats.style.Styler.text_gradient \
- pandas.DataFrame.values \
- pandas.DataFrame.loc \
- pandas.DataFrame.iloc \
- pandas.DataFrame.groupby \
- pandas.DataFrame.describe \
- pandas.DataFrame.skew \
- pandas.DataFrame.var \
- pandas.DataFrame.idxmax \
- pandas.DataFrame.idxmin \
- pandas.DataFrame.last \
- pandas.DataFrame.pivot \
- pandas.DataFrame.sort_values \
- pandas.DataFrame.tz_convert \
- pandas.DataFrame.tz_localize \
- pandas.DataFrame.plot.bar \
- pandas.DataFrame.plot.hexbin \
- pandas.DataFrame.plot.hist \
- pandas.DataFrame.plot.line \
- pandas.DataFrame.hist \
RET=$(($RET + $?)) ; echo $MSG "DONE"
fi
diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml
index 45f114322015b..ed7dfe1a3c17e 100644
--- a/ci/deps/actions-310.yaml
+++ b/ci/deps/actions-310.yaml
@@ -14,7 +14,6 @@ dependencies:
- pytest>=7.3.2
- pytest-cov
- pytest-xdist>=2.2.0
- - pytest-localserver>=0.7.1
- pytest-qt>=4.2.0
- boto3
@@ -27,7 +26,7 @@ dependencies:
- beautifulsoup4>=4.11.2
- blosc>=1.21.3
- bottleneck>=1.3.6
- - fastparquet>=2022.12.0
+ - fastparquet>=2023.10.0
- fsspec>=2022.11.0
- html5lib>=1.1
- hypothesis>=6.46.1
@@ -58,6 +57,7 @@ dependencies:
- zstandard>=0.19.0
- pip:
- - adbc-driver-postgresql>=0.8.0
+ - adbc-driver-postgresql>=0.10.0
- adbc-driver-sqlite>=0.8.0
- tzdata>=2022.7
+ - pytest-localserver>=0.7.1
diff --git a/ci/deps/actions-311-downstream_compat.yaml b/ci/deps/actions-311-downstream_compat.yaml
index d6bf9ec7843de..dd1d341c70a9b 100644
--- a/ci/deps/actions-311-downstream_compat.yaml
+++ b/ci/deps/actions-311-downstream_compat.yaml
@@ -28,7 +28,7 @@ dependencies:
- beautifulsoup4>=4.11.2
- blosc>=1.21.3
- bottleneck>=1.3.6
- - fastparquet>=2022.12.0
+ - fastparquet>=2023.10.0
- fsspec>=2022.11.0
- html5lib>=1.1
- hypothesis>=6.46.1
@@ -72,7 +72,6 @@ dependencies:
- pyyaml
- py
- pip:
- - adbc-driver-postgresql>=0.8.0
+ - adbc-driver-postgresql>=0.10.0
- adbc-driver-sqlite>=0.8.0
- - dataframe-api-compat>=0.1.7
- tzdata>=2022.7
diff --git a/ci/deps/actions-311-numpydev.yaml b/ci/deps/actions-311-numpydev.yaml
index b62e8630f2059..61a0eabbf133c 100644
--- a/ci/deps/actions-311-numpydev.yaml
+++ b/ci/deps/actions-311-numpydev.yaml
@@ -13,10 +13,7 @@ dependencies:
# test dependencies
- pytest>=7.3.2
- pytest-cov
- # Once pytest-cov > 4 comes out, unpin this
- # Right now, a DeprecationWarning related to rsyncdir
- # causes an InternalError within pytest
- - pytest-xdist>=2.2.0, <3
+ - pytest-xdist>=2.2.0
- hypothesis>=6.46.1
# pandas dependencies
diff --git a/ci/deps/actions-311-sanitizers.yaml b/ci/deps/actions-311-sanitizers.yaml
deleted file mode 100644
index f5f04c90bffad..0000000000000
--- a/ci/deps/actions-311-sanitizers.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-name: pandas-dev
-channels:
- - conda-forge
-dependencies:
- - python=3.11
-
- # build dependencies
- - versioneer[toml]
- - cython>=0.29.33
- - meson[ninja]=1.2.1
- - meson-python=0.13.1
-
- # test dependencies
- - pytest>=7.3.2
- - pytest-cov
- - pytest-xdist>=2.2.0
- - pytest-localserver>=0.7.1
- - pytest-qt>=4.2.0
- - boto3
- - hypothesis>=6.46.1
- - pyqt>=5.15.9
-
- # required dependencies
- - python-dateutil
- - numpy
- - pytz
-
- # pandas dependencies
- - pip
-
- - pip:
- - "tzdata>=2022.7"
diff --git a/ci/deps/actions-311.yaml b/ci/deps/actions-311.yaml
index d14686696e669..388116439f944 100644
--- a/ci/deps/actions-311.yaml
+++ b/ci/deps/actions-311.yaml
@@ -14,7 +14,6 @@ dependencies:
- pytest>=7.3.2
- pytest-cov
- pytest-xdist>=2.2.0
- - pytest-localserver>=0.7.1
- pytest-qt>=4.2.0
- boto3
@@ -27,7 +26,7 @@ dependencies:
- beautifulsoup4>=4.11.2
- blosc>=1.21.3
- bottleneck>=1.3.6
- - fastparquet>=2022.12.0
+ - fastparquet>=2023.10.0
- fsspec>=2022.11.0
- html5lib>=1.1
- hypothesis>=6.46.1
@@ -58,6 +57,6 @@ dependencies:
- zstandard>=0.19.0
- pip:
- - adbc-driver-postgresql>=0.8.0
+ - adbc-driver-postgresql>=0.10.0
- adbc-driver-sqlite>=0.8.0
- - tzdata>=2022.7
+ - pytest-localserver>=0.7.1
diff --git a/ci/deps/actions-312.yaml b/ci/deps/actions-312.yaml
index 86aaf24b4e15c..1d9f8aa3b092a 100644
--- a/ci/deps/actions-312.yaml
+++ b/ci/deps/actions-312.yaml
@@ -14,7 +14,6 @@ dependencies:
- pytest>=7.3.2
- pytest-cov
- pytest-xdist>=2.2.0
- - pytest-localserver>=0.7.1
- pytest-qt>=4.2.0
- boto3
@@ -27,7 +26,7 @@ dependencies:
- beautifulsoup4>=4.11.2
- blosc>=1.21.3
- bottleneck>=1.3.6
- - fastparquet>=2022.12.0
+ - fastparquet>=2023.10.0
- fsspec>=2022.11.0
- html5lib>=1.1
- hypothesis>=6.46.1
@@ -35,7 +34,7 @@ dependencies:
- jinja2>=3.1.2
- lxml>=4.9.2
- matplotlib>=3.6.3
- # - numba>=0.56.4
+ - numba>=0.56.4
- numexpr>=2.8.4
- odfpy>=1.4.1
- qtpy>=2.3.0
@@ -45,7 +44,7 @@ dependencies:
- pyarrow>=10.0.1
- pymysql>=1.0.2
- pyreadstat>=1.2.0
- # - pytables>=3.8.0
+ - pytables>=3.8.0
- python-calamine>=0.1.7
- pyxlsb>=1.0.10
- s3fs>=2022.11.0
@@ -58,6 +57,7 @@ dependencies:
- zstandard>=0.19.0
- pip:
- - adbc-driver-postgresql>=0.8.0
+ - adbc-driver-postgresql>=0.10.0
- adbc-driver-sqlite>=0.8.0
- tzdata>=2022.7
+ - pytest-localserver>=0.7.1
diff --git a/ci/deps/actions-39-minimum_versions.yaml b/ci/deps/actions-39-minimum_versions.yaml
index 7067048c4434d..b760f27a3d4d3 100644
--- a/ci/deps/actions-39-minimum_versions.yaml
+++ b/ci/deps/actions-39-minimum_versions.yaml
@@ -22,14 +22,14 @@ dependencies:
# required dependencies
- python-dateutil=2.8.2
- - numpy=1.22.4
+ - numpy=1.23.5
- pytz=2020.1
# optional dependencies
- beautifulsoup4=4.11.2
- blosc=1.21.3
- bottleneck=1.3.6
- - fastparquet=2022.12.0
+ - fastparquet=2023.10.0
- fsspec=2022.11.0
- html5lib=1.1
- hypothesis=6.46.1
@@ -60,7 +60,6 @@ dependencies:
- zstandard=0.19.0
- pip:
- - adbc-driver-postgresql==0.8.0
+ - adbc-driver-postgresql==0.10.0
- adbc-driver-sqlite==0.8.0
- - dataframe-api-compat==0.1.7
- tzdata==2022.7
diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml
index 31ee74174cd46..8f235a836bb3d 100644
--- a/ci/deps/actions-39.yaml
+++ b/ci/deps/actions-39.yaml
@@ -14,7 +14,6 @@ dependencies:
- pytest>=7.3.2
- pytest-cov
- pytest-xdist>=2.2.0
- - pytest-localserver>=0.7.1
- pytest-qt>=4.2.0
- boto3
@@ -27,7 +26,7 @@ dependencies:
- beautifulsoup4>=4.11.2
- blosc>=1.21.3
- bottleneck>=1.3.6
- - fastparquet>=2022.12.0
+ - fastparquet>=2023.10.0
- fsspec>=2022.11.0
- html5lib>=1.1
- hypothesis>=6.46.1
@@ -58,6 +57,7 @@ dependencies:
- zstandard>=0.19.0
- pip:
- - adbc-driver-postgresql>=0.8.0
+ - adbc-driver-postgresql>=0.10.0
- adbc-driver-sqlite>=0.8.0
- tzdata>=2022.7
+ - pytest-localserver>=0.7.1
diff --git a/ci/deps/circle-310-arm64.yaml b/ci/deps/circle-310-arm64.yaml
index a19ffd485262d..ed4d139714e71 100644
--- a/ci/deps/circle-310-arm64.yaml
+++ b/ci/deps/circle-310-arm64.yaml
@@ -27,7 +27,7 @@ dependencies:
- beautifulsoup4>=4.11.2
- blosc>=1.21.3
- bottleneck>=1.3.6
- - fastparquet>=2022.12.0
+ - fastparquet>=2023.10.0
- fsspec>=2022.11.0
- html5lib>=1.1
- hypothesis>=6.46.1
diff --git a/ci/run_tests.sh b/ci/run_tests.sh
index 48ef21686a26f..d2c2f58427a23 100755
--- a/ci/run_tests.sh
+++ b/ci/run_tests.sh
@@ -10,7 +10,7 @@ echo PYTHONHASHSEED=$PYTHONHASHSEED
COVERAGE="-s --cov=pandas --cov-report=xml --cov-append --cov-config=pyproject.toml"
-PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fEs -n $PYTEST_WORKERS --dist=loadfile $TEST_ARGS $COVERAGE $PYTEST_TARGET"
+PYTEST_CMD="MESONPY_EDITABLE_VERBOSE=1 PYTHONDEVMODE=1 PYTHONWARNDEFAULTENCODING=1 pytest -r fE -n $PYTEST_WORKERS --dist=worksteal $TEST_ARGS $COVERAGE $PYTEST_TARGET"
if [[ "$PATTERN" ]]; then
PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""
diff --git a/doc/make.py b/doc/make.py
index 2583242786fc8..02deb5002fea1 100755
--- a/doc/make.py
+++ b/doc/make.py
@@ -11,6 +11,7 @@
$ python make.py html
$ python make.py latex
"""
+
import argparse
import csv
import importlib
@@ -113,7 +114,7 @@ def _run_os(*args) -> None:
Examples
--------
- >>> DocBuilder()._run_os('python', '--version')
+ >>> DocBuilder()._run_os("python", "--version")
"""
subprocess.check_call(args, stdout=sys.stdout, stderr=sys.stderr)
@@ -129,7 +130,7 @@ def _sphinx_build(self, kind: str):
Examples
--------
- >>> DocBuilder(num_jobs=4)._sphinx_build('html')
+ >>> DocBuilder(num_jobs=4)._sphinx_build("html")
"""
if kind not in ("html", "latex", "linkcheck"):
raise ValueError(f"kind must be html, latex or linkcheck, not {kind}")
@@ -233,7 +234,7 @@ def html(self):
ret_code = self._sphinx_build("html")
zip_fname = os.path.join(BUILD_PATH, "html", "pandas.zip")
if os.path.exists(zip_fname):
- os.remove(zip_fname)
+ os.remove(zip_fname) # noqa: TID251
if ret_code == 0:
if self.single_doc_html is not None:
@@ -285,7 +286,7 @@ def zip_html(self) -> None:
"""
zip_fname = os.path.join(BUILD_PATH, "html", "pandas.zip")
if os.path.exists(zip_fname):
- os.remove(zip_fname)
+ os.remove(zip_fname) # noqa: TID251
dirname = os.path.join(BUILD_PATH, "html")
fnames = os.listdir(dirname)
os.chdir(dirname)
diff --git a/doc/redirects.csv b/doc/redirects.csv
index 27b41da63c513..e71a031bd67fd 100644
--- a/doc/redirects.csv
+++ b/doc/redirects.csv
@@ -118,7 +118,6 @@ generated/pandas.api.types.is_int64_dtype,../reference/api/pandas.api.types.is_i
generated/pandas.api.types.is_integer_dtype,../reference/api/pandas.api.types.is_integer_dtype
generated/pandas.api.types.is_integer,../reference/api/pandas.api.types.is_integer
generated/pandas.api.types.is_interval_dtype,../reference/api/pandas.api.types.is_interval_dtype
-generated/pandas.api.types.is_interval,../reference/api/pandas.api.types.is_interval
generated/pandas.api.types.is_iterator,../reference/api/pandas.api.types.is_iterator
generated/pandas.api.types.is_list_like,../reference/api/pandas.api.types.is_list_like
generated/pandas.api.types.is_named_tuple,../reference/api/pandas.api.types.is_named_tuple
@@ -239,7 +238,6 @@ generated/pandas.core.resample.Resampler.backfill,../reference/api/pandas.core.r
generated/pandas.core.resample.Resampler.bfill,../reference/api/pandas.core.resample.Resampler.bfill
generated/pandas.core.resample.Resampler.count,../reference/api/pandas.core.resample.Resampler.count
generated/pandas.core.resample.Resampler.ffill,../reference/api/pandas.core.resample.Resampler.ffill
-generated/pandas.core.resample.Resampler.fillna,../reference/api/pandas.core.resample.Resampler.fillna
generated/pandas.core.resample.Resampler.first,../reference/api/pandas.core.resample.Resampler.first
generated/pandas.core.resample.Resampler.get_group,../reference/api/pandas.core.resample.Resampler.get_group
generated/pandas.core.resample.Resampler.groups,../reference/api/pandas.core.resample.Resampler.groups
@@ -312,7 +310,6 @@ generated/pandas.DataFrame.align,../reference/api/pandas.DataFrame.align
generated/pandas.DataFrame.all,../reference/api/pandas.DataFrame.all
generated/pandas.DataFrame.any,../reference/api/pandas.DataFrame.any
generated/pandas.DataFrame.apply,../reference/api/pandas.DataFrame.apply
-generated/pandas.DataFrame.applymap,../reference/api/pandas.DataFrame.applymap
generated/pandas.DataFrame.as_blocks,../reference/api/pandas.DataFrame.as_blocks
generated/pandas.DataFrame.asfreq,../reference/api/pandas.DataFrame.asfreq
generated/pandas.DataFrame.as_matrix,../reference/api/pandas.DataFrame.as_matrix
@@ -325,7 +322,6 @@ generated/pandas.DataFrame.axes,../reference/api/pandas.DataFrame.axes
generated/pandas.DataFrame.between_time,../reference/api/pandas.DataFrame.between_time
generated/pandas.DataFrame.bfill,../reference/api/pandas.DataFrame.bfill
generated/pandas.DataFrame.blocks,../reference/api/pandas.DataFrame.blocks
-generated/pandas.DataFrame.bool,../reference/api/pandas.DataFrame.bool
generated/pandas.DataFrame.boxplot,../reference/api/pandas.DataFrame.boxplot
generated/pandas.DataFrame.clip,../reference/api/pandas.DataFrame.clip
generated/pandas.DataFrame.clip_lower,../reference/api/pandas.DataFrame.clip_lower
@@ -484,7 +480,6 @@ generated/pandas.DataFrame.style,../reference/api/pandas.DataFrame.style
generated/pandas.DataFrame.sub,../reference/api/pandas.DataFrame.sub
generated/pandas.DataFrame.subtract,../reference/api/pandas.DataFrame.subtract
generated/pandas.DataFrame.sum,../reference/api/pandas.DataFrame.sum
-generated/pandas.DataFrame.swapaxes,../reference/api/pandas.DataFrame.swapaxes
generated/pandas.DataFrame.swaplevel,../reference/api/pandas.DataFrame.swaplevel
generated/pandas.DataFrame.tail,../reference/api/pandas.DataFrame.tail
generated/pandas.DataFrame.take,../reference/api/pandas.DataFrame.take
@@ -495,7 +490,6 @@ generated/pandas.DataFrame.to_csv,../reference/api/pandas.DataFrame.to_csv
generated/pandas.DataFrame.to_dict,../reference/api/pandas.DataFrame.to_dict
generated/pandas.DataFrame.to_excel,../reference/api/pandas.DataFrame.to_excel
generated/pandas.DataFrame.to_feather,../reference/api/pandas.DataFrame.to_feather
-generated/pandas.DataFrame.to_gbq,../reference/api/pandas.DataFrame.to_gbq
generated/pandas.DataFrame.to_hdf,../reference/api/pandas.DataFrame.to_hdf
generated/pandas.DataFrame.to,../reference/api/pandas.DataFrame.to
generated/pandas.DataFrame.to_json,../reference/api/pandas.DataFrame.to_json
@@ -750,8 +744,6 @@ generated/pandas.Interval.overlaps,../reference/api/pandas.Interval.overlaps
generated/pandas.interval_range,../reference/api/pandas.interval_range
generated/pandas.Interval.right,../reference/api/pandas.Interval.right
generated/pandas.io.formats.style.Styler.apply,../reference/api/pandas.io.formats.style.Styler.apply
-generated/pandas.io.formats.style.Styler.applymap,../reference/api/pandas.io.formats.style.Styler.map
-generated/pandas.io.formats.style.Styler.applymap_index,../reference/api/pandas.io.formats.style.Styler.map_index
generated/pandas.io.formats.style.Styler.background_gradient,../reference/api/pandas.io.formats.style.Styler.background_gradient
generated/pandas.io.formats.style.Styler.bar,../reference/api/pandas.io.formats.style.Styler.bar
generated/pandas.io.formats.style.Styler.clear,../reference/api/pandas.io.formats.style.Styler.clear
@@ -890,7 +882,6 @@ generated/pandas.read_csv,../reference/api/pandas.read_csv
generated/pandas.read_excel,../reference/api/pandas.read_excel
generated/pandas.read_feather,../reference/api/pandas.read_feather
generated/pandas.read_fwf,../reference/api/pandas.read_fwf
-generated/pandas.read_gbq,../reference/api/pandas.read_gbq
generated/pandas.read_hdf,../reference/api/pandas.read_hdf
generated/pandas.read,../reference/api/pandas.read
generated/pandas.read_json,../reference/api/pandas.read_json
@@ -932,7 +923,6 @@ generated/pandas.Series.between,../reference/api/pandas.Series.between
generated/pandas.Series.between_time,../reference/api/pandas.Series.between_time
generated/pandas.Series.bfill,../reference/api/pandas.Series.bfill
generated/pandas.Series.blocks,../reference/api/pandas.Series.blocks
-generated/pandas.Series.bool,../reference/api/pandas.Series.bool
generated/pandas.Series.cat.add_categories,../reference/api/pandas.Series.cat.add_categories
generated/pandas.Series.cat.as_ordered,../reference/api/pandas.Series.cat.as_ordered
generated/pandas.Series.cat.as_unordered,../reference/api/pandas.Series.cat.as_unordered
@@ -1116,7 +1106,6 @@ generated/pandas.Series.ptp,../reference/api/pandas.Series.ptp
generated/pandas.Series.quantile,../reference/api/pandas.Series.quantile
generated/pandas.Series.radd,../reference/api/pandas.Series.radd
generated/pandas.Series.rank,../reference/api/pandas.Series.rank
-generated/pandas.Series.ravel,../reference/api/pandas.Series.ravel
generated/pandas.Series.rdiv,../reference/api/pandas.Series.rdiv
generated/pandas.Series.rdivmod,../reference/api/pandas.Series.rdivmod
generated/pandas.Series.real,../reference/api/pandas.Series.real
@@ -1215,7 +1204,6 @@ generated/pandas.Series.str.zfill,../reference/api/pandas.Series.str.zfill
generated/pandas.Series.sub,../reference/api/pandas.Series.sub
generated/pandas.Series.subtract,../reference/api/pandas.Series.subtract
generated/pandas.Series.sum,../reference/api/pandas.Series.sum
-generated/pandas.Series.swapaxes,../reference/api/pandas.Series.swapaxes
generated/pandas.Series.swaplevel,../reference/api/pandas.Series.swaplevel
generated/pandas.Series.tail,../reference/api/pandas.Series.tail
generated/pandas.Series.take,../reference/api/pandas.Series.take
@@ -1251,7 +1239,6 @@ generated/pandas.Series.valid,../reference/api/pandas.Series.valid
generated/pandas.Series.value_counts,../reference/api/pandas.Series.value_counts
generated/pandas.Series.values,../reference/api/pandas.Series.values
generated/pandas.Series.var,../reference/api/pandas.Series.var
-generated/pandas.Series.view,../reference/api/pandas.Series.view
generated/pandas.Series.where,../reference/api/pandas.Series.where
generated/pandas.Series.xs,../reference/api/pandas.Series.xs
generated/pandas.set_option,../reference/api/pandas.set_option
@@ -1439,8 +1426,6 @@ reference/api/pandas.arrays.PandasArray,pandas.arrays.NumpyExtensionArray
reference/api/pandas.core.groupby.DataFrameGroupBy.backfill,pandas.core.groupby.DataFrameGroupBy.bfill
reference/api/pandas.core.groupby.GroupBy.backfill,pandas.core.groupby.DataFrameGroupBy.bfill
reference/api/pandas.core.resample.Resampler.backfill,pandas.core.resample.Resampler.bfill
-reference/api/pandas.io.formats.style.Styler.applymap,pandas.io.formats.style.Styler.map
-reference/api/pandas.io.formats.style.Styler.applymap_index,pandas.io.formats.style.Styler.map_index
# EWM -> ExponentialMovingWindow
reference/api/pandas.core.window.ewm.EWM.corr,pandas.core.window.ewm.ExponentialMovingWindow.corr
diff --git a/doc/scripts/eval_performance.py b/doc/scripts/eval_performance.py
index dded8362a4cda..0383d4d598d55 100644
--- a/doc/scripts/eval_performance.py
+++ b/doc/scripts/eval_performance.py
@@ -17,7 +17,7 @@ def bench_with(n, times=10, repeat=3, engine="numexpr"):
return (
np.array(
timeit(
- f"df.eval(s, engine={repr(engine)})",
+ f"df.eval(s, engine={engine!r})",
setup=setup_common % (n, setup_with),
repeat=repeat,
number=times,
@@ -34,7 +34,7 @@ def bench_subset(n, times=20, repeat=3, engine="numexpr"):
return (
np.array(
timeit(
- f"df.query(s, engine={repr(engine)})",
+ f"df.query(s, engine={engine!r})",
setup=setup_common % (n, setup_subset),
repeat=repeat,
number=times,
@@ -55,7 +55,7 @@ def bench(mn=3, mx=7, num=100, engines=("python", "numexpr"), verbose=False):
for engine in engines:
for i, n in enumerate(r):
if verbose & (i % 10 == 0):
- print(f"engine: {repr(engine)}, i == {i:d}")
+ print(f"engine: {engine!r}, i == {i:d}")
ev_times = bench_with(n, times=1, repeat=1, engine=engine)
ev.loc[i, engine] = np.mean(ev_times)
qu_times = bench_subset(n, times=1, repeat=1, engine=engine)
diff --git a/doc/source/_static/css/getting_started.css b/doc/source/_static/css/getting_started.css
index 0d53bbde94ae3..b02311eb66080 100644
--- a/doc/source/_static/css/getting_started.css
+++ b/doc/source/_static/css/getting_started.css
@@ -248,6 +248,7 @@ ul.task-bullet > li > p:first-child {
}
.tutorial-card .card-header {
+ --bs-card-cap-color: var(--pst-color-text-base);
cursor: pointer;
background-color: var(--pst-color-surface);
border: 1px solid var(--pst-color-border)
@@ -269,7 +270,7 @@ ul.task-bullet > li > p:first-child {
.tutorial-card .gs-badge-link a {
- color: var(--pst-color-text-base);
+ color: var(--pst-color-primary-text);
text-decoration: none;
}
diff --git a/doc/source/conf.py b/doc/source/conf.py
index be6150d4e54ba..77dd5d03d311c 100644
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -323,7 +323,6 @@
("pandas.io.clipboard.read_clipboard", "pandas.read_clipboard"),
("pandas.io.excel.ExcelFile.parse", "pandas.ExcelFile.parse"),
("pandas.io.excel.read_excel", "pandas.read_excel"),
- ("pandas.io.gbq.read_gbq", "pandas.read_gbq"),
("pandas.io.html.read_html", "pandas.read_html"),
("pandas.io.json.read_json", "pandas.read_json"),
("pandas.io.parsers.read_csv", "pandas.read_csv"),
@@ -432,7 +431,7 @@
"index",
"pandas.tex",
"pandas: powerful Python data analysis toolkit",
- "Wes McKinney and the Pandas Development Team",
+ "Wes McKinney and the pandas Development Team",
"manual",
)
]
@@ -460,7 +459,6 @@
"dateutil": ("https://dateutil.readthedocs.io/en/latest/", None),
"matplotlib": ("https://matplotlib.org/stable/", None),
"numpy": ("https://numpy.org/doc/stable/", None),
- "py": ("https://pylib.readthedocs.io/en/latest/", None),
"python": ("https://docs.python.org/3/", None),
"scipy": ("https://docs.scipy.org/doc/scipy/", None),
"pyarrow": ("https://arrow.apache.org/docs/", None),
@@ -813,8 +811,6 @@ def setup(app) -> None:
for link in [
"http://scatterci.github.io/pydata/pandas",
"http://specs.frictionlessdata.io/json-table-schema/",
- "https://cloud.google.com/bigquery/docs/access-control#roles",
- "https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query",
"https://crates.io/crates/calamine",
"https://devguide.python.org/setup/#macos",
"https://en.wikipedia.org/wiki/Imputation_statistics",
@@ -830,7 +826,6 @@ def setup(app) -> None:
"https://nipunbatra.github.io/blog/visualisation/2013/05/01/aggregation-timeseries.html",
"https://nbviewer.ipython.org/gist/metakermit/5720498",
"https://numpy.org/doc/stable/user/basics.byteswapping.html",
- "https://pandas-gbq.readthedocs.io/en/latest/changelog.html#changelog-0-8-0",
"https://pandas.pydata.org/pandas-docs/stable/io.html#io-chunking",
"https://pandas.pydata.org/pandas-docs/stable/ecosystem.html",
"https://sqlalchemy.readthedocs.io/en/latest/dialects/index.html",
diff --git a/doc/source/development/community.rst b/doc/source/development/community.rst
index 9a4de3c2580ab..ccf7be8e47748 100644
--- a/doc/source/development/community.rst
+++ b/doc/source/development/community.rst
@@ -112,7 +112,7 @@ people who are hesitant to bring up their questions or ideas on a large public
mailing list or GitHub.
If this sounds like the right place for you, you are welcome to join using
-`this link `_!
+`this link `_!
Please remember to follow our `Code of Conduct `_,
and be aware that our admins are monitoring for irrelevant messages and will remove folks who use
our
diff --git a/doc/source/development/contributing.rst b/doc/source/development/contributing.rst
index 82af8122a6bbd..78d22c768b865 100644
--- a/doc/source/development/contributing.rst
+++ b/doc/source/development/contributing.rst
@@ -19,7 +19,7 @@ Bug reports and enhancement requests
====================================
Bug reports and enhancement requests are an important part of making pandas more stable and
-are curated though Github issues. When reporting and issue or request, please select the `appropriate
+are curated though Github issues. When reporting an issue or request, please select the `appropriate
category and fill out the issue form fully `_
to ensure others and the core development team can fully understand the scope of the issue.
diff --git a/doc/source/development/contributing_codebase.rst b/doc/source/development/contributing_codebase.rst
index be8249cd3a287..39e279fd5c917 100644
--- a/doc/source/development/contributing_codebase.rst
+++ b/doc/source/development/contributing_codebase.rst
@@ -38,7 +38,7 @@ Pre-commit
----------
Additionally, :ref:`Continuous Integration ` will run code formatting checks
-like ``black``, ``ruff``,
+like ``ruff``,
``isort``, and ``clang-format`` and more using `pre-commit hooks ` to fail; therefore,
it is helpful to run the check yourself before submitting code. This
@@ -253,7 +253,7 @@ Testing type hints in code using pandas
.. warning::
- * Pandas is not yet a py.typed library (:pep:`561`)!
+ * pandas is not yet a py.typed library (:pep:`561`)!
The primary purpose of locally declaring pandas as a py.typed library is to test and
improve the pandas-builtin type annotations.
@@ -596,14 +596,15 @@ with the specific exception subclass (i.e. never use :py:class:`Exception`) and
Testing involving files
^^^^^^^^^^^^^^^^^^^^^^^
-The ``tm.ensure_clean`` context manager creates a temporary file for testing,
-with a generated filename (or your filename if provided), that is automatically
-deleted when the context block is exited.
+The ``temp_file`` pytest fixture creates a temporary file :py:class:`Pathlib` object for testing:
.. code-block:: python
- with tm.ensure_clean('my_file_path') as path:
- # do something with the path
+ def test_something(temp_file):
+ pd.DataFrame([1]).to_csv(str(temp_file))
+
+Please reference `pytest's documentation `_
+for the file retension policy.
Testing involving network connectivity
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/doc/source/development/contributing_docstring.rst b/doc/source/development/contributing_docstring.rst
index 87aecb6936c9c..0b8c1e16dce0e 100644
--- a/doc/source/development/contributing_docstring.rst
+++ b/doc/source/development/contributing_docstring.rst
@@ -939,8 +939,8 @@ Each shared docstring will have a base template with variables, like
Finally, docstrings can also be appended to with the ``doc`` decorator.
In this example, we'll create a parent docstring normally (this is like
-``pandas.core.generic.NDFrame``. Then we'll have two children (like
-``pandas.core.series.Series`` and ``pandas.core.frame.DataFrame``). We'll
+``pandas.core.generic.NDFrame``). Then we'll have two children (like
+``pandas.core.series.Series`` and ``pandas.DataFrame``). We'll
substitute the class names in this docstring.
.. code-block:: python
diff --git a/doc/source/development/contributing_documentation.rst b/doc/source/development/contributing_documentation.rst
index 964f82be4fa7b..443470e6c50f9 100644
--- a/doc/source/development/contributing_documentation.rst
+++ b/doc/source/development/contributing_documentation.rst
@@ -14,7 +14,7 @@ experts. If something in the docs doesn't make sense to you, updating the
relevant section after you figure it out is a great way to ensure it will help
the next person. Please visit the `issues page `__
for a full list of issues that are currently open regarding the
-Pandas documentation.
+pandas documentation.
diff --git a/doc/source/development/debugging_extensions.rst b/doc/source/development/debugging_extensions.rst
index d63ecb3157cff..f09d73fa13b9a 100644
--- a/doc/source/development/debugging_extensions.rst
+++ b/doc/source/development/debugging_extensions.rst
@@ -6,7 +6,7 @@
Debugging C extensions
======================
-Pandas uses Cython and C/C++ `extension modules `_ to optimize performance. Unfortunately, the standard Python debugger does not allow you to step into these extensions. Cython extensions can be debugged with the `Cython debugger `_ and C/C++ extensions can be debugged using the tools shipped with your platform's compiler.
+pandas uses Cython and C/C++ `extension modules `_ to optimize performance. Unfortunately, the standard Python debugger does not allow you to step into these extensions. Cython extensions can be debugged with the `Cython debugger `_ and C/C++ extensions can be debugged using the tools shipped with your platform's compiler.
For Python developers with limited or no C/C++ experience this can seem a daunting task. Core developer Will Ayd has written a 3 part blog series to help guide you from the standard Python debugger into these other tools:
diff --git a/doc/source/development/maintaining.rst b/doc/source/development/maintaining.rst
index c7803d8401e4e..f6ff95aa72c6c 100644
--- a/doc/source/development/maintaining.rst
+++ b/doc/source/development/maintaining.rst
@@ -151,7 +151,7 @@ and then run::
git bisect start
git bisect good v1.4.0
git bisect bad v1.5.0
- git bisect run bash -c "python setup.py build_ext -j 4; python t.py"
+ git bisect run bash -c "python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true; python t.py"
This finds the first commit that changed the behavior. The C extensions have to be
rebuilt at every step, so the search can take a while.
@@ -159,7 +159,7 @@ rebuilt at every step, so the search can take a while.
Exit bisect and rebuild the current version::
git bisect reset
- python setup.py build_ext -j 4
+ python -m pip install -ve . --no-build-isolation --config-settings editable-verbose=true
Report your findings under the corresponding issue and ping the commit author to get
their input.
@@ -326,34 +326,6 @@ a milestone before tagging, you can request the bot to backport it with:
@Meeseeksdev backport
-.. _maintaining.asv-machine:
-
-Benchmark machine
------------------
-
-The team currently owns dedicated hardware for hosting a website for pandas' ASV performance benchmark. The results
-are published to https://asv-runner.github.io/asv-collection/pandas/
-
-Configuration
-`````````````
-
-The machine can be configured with the `Ansible `_ playbook in https://github.com/tomaugspurger/asv-runner.
-
-Publishing
-``````````
-
-The results are published to another GitHub repository, https://github.com/tomaugspurger/asv-collection.
-Finally, we have a cron job on our docs server to pull from https://github.com/tomaugspurger/asv-collection, to serve them from ``/speed``.
-Ask Tom or Joris for access to the webserver.
-
-Debugging
-`````````
-
-The benchmarks are scheduled by Airflow. It has a dashboard for viewing and debugging the results. You'll need to setup an SSH tunnel to view them
-
- ssh -L 8080:localhost:8080 pandas@panda.likescandy.com
-
-
.. _maintaining.release:
Release process
@@ -430,7 +402,7 @@ Release
git checkout
git pull --ff-only upstream
git clean -xdf
- git commit --allow-empty --author="Pandas Development Team " -m "RLS: "
+ git commit --allow-empty --author="pandas Development Team " -m "RLS: "
git tag -a v -m "Version " # NOTE that the tag is v1.5.2 with "v" not 1.5.2
git push upstream --follow-tags
@@ -460,7 +432,7 @@ which will be triggered when the tag is pushed.
4. Create a `new GitHub release `_:
- Tag: ````
- - Title: ``Pandas ``
+ - Title: ``pandas ``
- Description: Copy the description of the last release of the same kind (release candidate, major/minor or patch release)
- Files: ``pandas-.tar.gz`` source distribution just generated
- Set as a pre-release: Only check for a release candidate
diff --git a/doc/source/getting_started/comparison/comparison_with_r.rst b/doc/source/getting_started/comparison/comparison_with_r.rst
index a6cfcd4614984..25ba237e8caf3 100644
--- a/doc/source/getting_started/comparison/comparison_with_r.rst
+++ b/doc/source/getting_started/comparison/comparison_with_r.rst
@@ -438,7 +438,7 @@ In Python, the :meth:`~pandas.melt` method is the R equivalent:
)
pd.melt(cheese, id_vars=["first", "last"])
- cheese.set_index(["first", "last"]).stack(future_stack=True) # alternative way
+ cheese.set_index(["first", "last"]).stack() # alternative way
For more details and examples see :ref:`the reshaping documentation
`.
diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst
index 1d7eca5223544..3cd9e030d6b3c 100644
--- a/doc/source/getting_started/install.rst
+++ b/doc/source/getting_started/install.rst
@@ -21,7 +21,7 @@ Instructions for installing :ref:`from source `,
Python version support
----------------------
-Officially Python 3.9, 3.10 and 3.11.
+Officially Python 3.9, 3.10, 3.11 and 3.12.
Installing pandas
-----------------
@@ -203,7 +203,7 @@ pandas requires the following dependencies.
================================================================ ==========================
Package Minimum supported version
================================================================ ==========================
-`NumPy `__ 1.22.4
+`NumPy `__ 1.23.5
`python-dateutil ` with column names, whereas
- the outer brackets are used to select the data from a pandas
+ the outer square brackets are used to select the data from a pandas
``DataFrame`` as seen in the previous example.
The returned data type is a pandas DataFrame:
@@ -354,7 +354,7 @@ See the user guide section on :ref:`different choices for indexing REMEMBER
- When selecting subsets of data, square brackets ``[]`` are used.
-- Inside these brackets, you can use a single column/row label, a list
+- Inside these square brackets, you can use a single column/row label, a list
of column/row labels, a slice of labels, a conditional expression or
a colon.
- Select specific rows and/or columns using ``loc`` when using the row
diff --git a/doc/source/getting_started/intro_tutorials/05_add_columns.rst b/doc/source/getting_started/intro_tutorials/05_add_columns.rst
index d59a70cc2818e..3e0f75b210dbb 100644
--- a/doc/source/getting_started/intro_tutorials/05_add_columns.rst
+++ b/doc/source/getting_started/intro_tutorials/05_add_columns.rst
@@ -51,7 +51,7 @@ hPa, the conversion factor is 1.882*)
air_quality["london_mg_per_cubic"] = air_quality["station_london"] * 1.882
air_quality.head()
-To create a new column, use the ``[]`` brackets with the new column name
+To create a new column, use the square brackets ``[]`` with the new column name
at the left side of the assignment.
.. raw:: html
diff --git a/doc/source/getting_started/intro_tutorials/06_calculate_statistics.rst b/doc/source/getting_started/intro_tutorials/06_calculate_statistics.rst
index fe3ae820e7085..668925ce79252 100644
--- a/doc/source/getting_started/intro_tutorials/06_calculate_statistics.rst
+++ b/doc/source/getting_started/intro_tutorials/06_calculate_statistics.rst
@@ -162,7 +162,7 @@ columns by passing ``numeric_only=True``:
It does not make much sense to get the average value of the ``Pclass``.
If we are only interested in the average age for each gender, the
-selection of columns (rectangular brackets ``[]`` as usual) is supported
+selection of columns (square brackets ``[]`` as usual) is supported
on the grouped data as well:
.. ipython:: python
diff --git a/doc/source/reference/arrays.rst b/doc/source/reference/arrays.rst
index fe65364896f54..a631cd517e3c2 100644
--- a/doc/source/reference/arrays.rst
+++ b/doc/source/reference/arrays.rst
@@ -539,6 +539,21 @@ To create a Series of dtype ``category``, use ``cat = s.astype(dtype)`` or
If the :class:`Series` is of dtype :class:`CategoricalDtype`, ``Series.cat`` can be used to change the categorical
data. See :ref:`api.series.cat` for more.
+More methods are available on :class:`Categorical`:
+
+.. autosummary::
+ :toctree: api/
+
+ Categorical.as_ordered
+ Categorical.as_unordered
+ Categorical.set_categories
+ Categorical.rename_categories
+ Categorical.reorder_categories
+ Categorical.add_categories
+ Categorical.remove_categories
+ Categorical.remove_unused_categories
+ Categorical.map
+
.. _api.arrays.sparse:
Sparse
@@ -685,7 +700,6 @@ Scalar introspection
api.types.is_float
api.types.is_hashable
api.types.is_integer
- api.types.is_interval
api.types.is_number
api.types.is_re
api.types.is_re_compilable
diff --git a/doc/source/reference/frame.rst b/doc/source/reference/frame.rst
index fefb02dd916cd..7680c8b434866 100644
--- a/doc/source/reference/frame.rst
+++ b/doc/source/reference/frame.rst
@@ -48,7 +48,7 @@ Conversion
DataFrame.convert_dtypes
DataFrame.infer_objects
DataFrame.copy
- DataFrame.bool
+ DataFrame.to_numpy
Indexing, iteration
~~~~~~~~~~~~~~~~~~~
@@ -74,6 +74,7 @@ Indexing, iteration
DataFrame.where
DataFrame.mask
DataFrame.query
+ DataFrame.isetitem
For more information on ``.at``, ``.iat``, ``.loc``, and
``.iloc``, see the :ref:`indexing documentation `.
@@ -117,7 +118,6 @@ Function application, GroupBy & window
DataFrame.apply
DataFrame.map
- DataFrame.applymap
DataFrame.pipe
DataFrame.agg
DataFrame.aggregate
@@ -185,11 +185,9 @@ Reindexing / selection / label manipulation
DataFrame.duplicated
DataFrame.equals
DataFrame.filter
- DataFrame.first
DataFrame.head
DataFrame.idxmax
DataFrame.idxmin
- DataFrame.last
DataFrame.reindex
DataFrame.reindex_like
DataFrame.rename
@@ -209,7 +207,6 @@ Missing data handling
.. autosummary::
:toctree: api/
- DataFrame.backfill
DataFrame.bfill
DataFrame.dropna
DataFrame.ffill
@@ -219,7 +216,6 @@ Missing data handling
DataFrame.isnull
DataFrame.notna
DataFrame.notnull
- DataFrame.pad
DataFrame.replace
Reshaping, sorting, transposing
@@ -238,7 +234,6 @@ Reshaping, sorting, transposing
DataFrame.swaplevel
DataFrame.stack
DataFrame.unstack
- DataFrame.swapaxes
DataFrame.melt
DataFrame.explode
DataFrame.squeeze
@@ -382,7 +377,6 @@ Serialization / IO / conversion
DataFrame.to_feather
DataFrame.to_latex
DataFrame.to_stata
- DataFrame.to_gbq
DataFrame.to_records
DataFrame.to_string
DataFrame.to_clipboard
diff --git a/doc/source/reference/groupby.rst b/doc/source/reference/groupby.rst
index 771163ae1b0bc..3b02ffe20c10e 100644
--- a/doc/source/reference/groupby.rst
+++ b/doc/source/reference/groupby.rst
@@ -80,7 +80,6 @@ Function application
DataFrameGroupBy.describe
DataFrameGroupBy.diff
DataFrameGroupBy.ffill
- DataFrameGroupBy.fillna
DataFrameGroupBy.first
DataFrameGroupBy.head
DataFrameGroupBy.idxmax
@@ -131,7 +130,6 @@ Function application
SeriesGroupBy.describe
SeriesGroupBy.diff
SeriesGroupBy.ffill
- SeriesGroupBy.fillna
SeriesGroupBy.first
SeriesGroupBy.head
SeriesGroupBy.last
diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst
index 7da02f7958416..639bac4d40b70 100644
--- a/doc/source/reference/index.rst
+++ b/doc/source/reference/index.rst
@@ -24,13 +24,14 @@ The following subpackages are public.
`pandas-stubs `_ package
which has classes in addition to those that occur in pandas for type-hinting.
-In addition, public functions in ``pandas.io`` and ``pandas.tseries`` submodules
-are mentioned in the documentation.
+In addition, public functions in ``pandas.io``, ``pandas.tseries``, ``pandas.util`` submodules
+are explicitly mentioned in the documentation. Further APIs in these modules are not guaranteed
+to be stable.
.. warning::
- The ``pandas.core``, ``pandas.compat``, and ``pandas.util`` top-level modules are PRIVATE. Stable functionality in such modules is not guaranteed.
+ The ``pandas.core``, ``pandas.compat`` top-level modules are PRIVATE. Stable functionality in such modules is not guaranteed.
.. If you update this toctree, also update the manual toctree in the
.. main index.rst.template
@@ -61,7 +62,6 @@ are mentioned in the documentation.
..
.. toctree::
- api/pandas.Index.holds_integer
api/pandas.Index.nlevels
api/pandas.Index.sort
diff --git a/doc/source/reference/indexing.rst b/doc/source/reference/indexing.rst
index fa6105761df0a..7a4bc0f467f9a 100644
--- a/doc/source/reference/indexing.rst
+++ b/doc/source/reference/indexing.rst
@@ -41,6 +41,7 @@ Properties
Index.empty
Index.T
Index.memory_usage
+ Index.array
Modifying and computations
~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -61,13 +62,6 @@ Modifying and computations
Index.identical
Index.insert
Index.is_
- Index.is_boolean
- Index.is_categorical
- Index.is_floating
- Index.is_integer
- Index.is_interval
- Index.is_numeric
- Index.is_object
Index.min
Index.max
Index.reindex
@@ -110,6 +104,7 @@ Conversion
Index.to_list
Index.to_series
Index.to_frame
+ Index.to_numpy
Index.view
Sorting
diff --git a/doc/source/reference/io.rst b/doc/source/reference/io.rst
index fbd0f6bd200b9..805fb8b783459 100644
--- a/doc/source/reference/io.rst
+++ b/doc/source/reference/io.rst
@@ -188,13 +188,6 @@ SQL
read_sql
DataFrame.to_sql
-Google BigQuery
-~~~~~~~~~~~~~~~
-.. autosummary::
- :toctree: api/
-
- read_gbq
-
STATA
~~~~~
.. autosummary::
diff --git a/doc/source/reference/offset_frequency.rst b/doc/source/reference/offset_frequency.rst
index ab89fe74e7337..8bb2c6ffe73be 100644
--- a/doc/source/reference/offset_frequency.rst
+++ b/doc/source/reference/offset_frequency.rst
@@ -26,8 +26,6 @@ Properties
DateOffset.normalize
DateOffset.rule_code
DateOffset.n
- DateOffset.is_month_start
- DateOffset.is_month_end
Methods
~~~~~~~
@@ -35,7 +33,6 @@ Methods
:toctree: api/
DateOffset.copy
- DateOffset.is_anchored
DateOffset.is_on_offset
DateOffset.is_month_start
DateOffset.is_month_end
@@ -82,7 +79,6 @@ Methods
:toctree: api/
BusinessDay.copy
- BusinessDay.is_anchored
BusinessDay.is_on_offset
BusinessDay.is_month_start
BusinessDay.is_month_end
@@ -122,7 +118,6 @@ Methods
:toctree: api/
BusinessHour.copy
- BusinessHour.is_anchored
BusinessHour.is_on_offset
BusinessHour.is_month_start
BusinessHour.is_month_end
@@ -169,7 +164,6 @@ Methods
:toctree: api/
CustomBusinessDay.copy
- CustomBusinessDay.is_anchored
CustomBusinessDay.is_on_offset
CustomBusinessDay.is_month_start
CustomBusinessDay.is_month_end
@@ -209,7 +203,6 @@ Methods
:toctree: api/
CustomBusinessHour.copy
- CustomBusinessHour.is_anchored
CustomBusinessHour.is_on_offset
CustomBusinessHour.is_month_start
CustomBusinessHour.is_month_end
@@ -244,7 +237,6 @@ Methods
:toctree: api/
MonthEnd.copy
- MonthEnd.is_anchored
MonthEnd.is_on_offset
MonthEnd.is_month_start
MonthEnd.is_month_end
@@ -279,7 +271,6 @@ Methods
:toctree: api/
MonthBegin.copy
- MonthBegin.is_anchored
MonthBegin.is_on_offset
MonthBegin.is_month_start
MonthBegin.is_month_end
@@ -323,7 +314,6 @@ Methods
:toctree: api/
BusinessMonthEnd.copy
- BusinessMonthEnd.is_anchored
BusinessMonthEnd.is_on_offset
BusinessMonthEnd.is_month_start
BusinessMonthEnd.is_month_end
@@ -367,7 +357,6 @@ Methods
:toctree: api/
BusinessMonthBegin.copy
- BusinessMonthBegin.is_anchored
BusinessMonthBegin.is_on_offset
BusinessMonthBegin.is_month_start
BusinessMonthBegin.is_month_end
@@ -415,7 +404,6 @@ Methods
:toctree: api/
CustomBusinessMonthEnd.copy
- CustomBusinessMonthEnd.is_anchored
CustomBusinessMonthEnd.is_on_offset
CustomBusinessMonthEnd.is_month_start
CustomBusinessMonthEnd.is_month_end
@@ -463,7 +451,6 @@ Methods
:toctree: api/
CustomBusinessMonthBegin.copy
- CustomBusinessMonthBegin.is_anchored
CustomBusinessMonthBegin.is_on_offset
CustomBusinessMonthBegin.is_month_start
CustomBusinessMonthBegin.is_month_end
@@ -499,7 +486,6 @@ Methods
:toctree: api/
SemiMonthEnd.copy
- SemiMonthEnd.is_anchored
SemiMonthEnd.is_on_offset
SemiMonthEnd.is_month_start
SemiMonthEnd.is_month_end
@@ -535,7 +521,6 @@ Methods
:toctree: api/
SemiMonthBegin.copy
- SemiMonthBegin.is_anchored
SemiMonthBegin.is_on_offset
SemiMonthBegin.is_month_start
SemiMonthBegin.is_month_end
@@ -571,7 +556,6 @@ Methods
:toctree: api/
Week.copy
- Week.is_anchored
Week.is_on_offset
Week.is_month_start
Week.is_month_end
@@ -607,7 +591,6 @@ Methods
:toctree: api/
WeekOfMonth.copy
- WeekOfMonth.is_anchored
WeekOfMonth.is_on_offset
WeekOfMonth.weekday
WeekOfMonth.is_month_start
@@ -645,7 +628,6 @@ Methods
:toctree: api/
LastWeekOfMonth.copy
- LastWeekOfMonth.is_anchored
LastWeekOfMonth.is_on_offset
LastWeekOfMonth.is_month_start
LastWeekOfMonth.is_month_end
@@ -681,7 +663,6 @@ Methods
:toctree: api/
BQuarterEnd.copy
- BQuarterEnd.is_anchored
BQuarterEnd.is_on_offset
BQuarterEnd.is_month_start
BQuarterEnd.is_month_end
@@ -717,7 +698,6 @@ Methods
:toctree: api/
BQuarterBegin.copy
- BQuarterBegin.is_anchored
BQuarterBegin.is_on_offset
BQuarterBegin.is_month_start
BQuarterBegin.is_month_end
@@ -753,7 +733,6 @@ Methods
:toctree: api/
QuarterEnd.copy
- QuarterEnd.is_anchored
QuarterEnd.is_on_offset
QuarterEnd.is_month_start
QuarterEnd.is_month_end
@@ -789,7 +768,6 @@ Methods
:toctree: api/
QuarterBegin.copy
- QuarterBegin.is_anchored
QuarterBegin.is_on_offset
QuarterBegin.is_month_start
QuarterBegin.is_month_end
@@ -825,7 +803,6 @@ Methods
:toctree: api/
BYearEnd.copy
- BYearEnd.is_anchored
BYearEnd.is_on_offset
BYearEnd.is_month_start
BYearEnd.is_month_end
@@ -861,7 +838,6 @@ Methods
:toctree: api/
BYearBegin.copy
- BYearBegin.is_anchored
BYearBegin.is_on_offset
BYearBegin.is_month_start
BYearBegin.is_month_end
@@ -897,7 +873,6 @@ Methods
:toctree: api/
YearEnd.copy
- YearEnd.is_anchored
YearEnd.is_on_offset
YearEnd.is_month_start
YearEnd.is_month_end
@@ -933,7 +908,6 @@ Methods
:toctree: api/
YearBegin.copy
- YearBegin.is_anchored
YearBegin.is_on_offset
YearBegin.is_month_start
YearBegin.is_month_end
@@ -973,7 +947,6 @@ Methods
FY5253.copy
FY5253.get_rule_code_suffix
FY5253.get_year_end
- FY5253.is_anchored
FY5253.is_on_offset
FY5253.is_month_start
FY5253.is_month_end
@@ -1014,7 +987,6 @@ Methods
FY5253Quarter.copy
FY5253Quarter.get_rule_code_suffix
FY5253Quarter.get_weeks
- FY5253Quarter.is_anchored
FY5253Quarter.is_on_offset
FY5253Quarter.year_has_extra_week
FY5253Quarter.is_month_start
@@ -1050,7 +1022,6 @@ Methods
:toctree: api/
Easter.copy
- Easter.is_anchored
Easter.is_on_offset
Easter.is_month_start
Easter.is_month_end
@@ -1071,7 +1042,6 @@ Properties
.. autosummary::
:toctree: api/
- Tick.delta
Tick.freqstr
Tick.kwds
Tick.name
@@ -1086,7 +1056,6 @@ Methods
:toctree: api/
Tick.copy
- Tick.is_anchored
Tick.is_on_offset
Tick.is_month_start
Tick.is_month_end
@@ -1107,7 +1076,6 @@ Properties
.. autosummary::
:toctree: api/
- Day.delta
Day.freqstr
Day.kwds
Day.name
@@ -1122,7 +1090,6 @@ Methods
:toctree: api/
Day.copy
- Day.is_anchored
Day.is_on_offset
Day.is_month_start
Day.is_month_end
@@ -1143,7 +1110,6 @@ Properties
.. autosummary::
:toctree: api/
- Hour.delta
Hour.freqstr
Hour.kwds
Hour.name
@@ -1158,7 +1124,6 @@ Methods
:toctree: api/
Hour.copy
- Hour.is_anchored
Hour.is_on_offset
Hour.is_month_start
Hour.is_month_end
@@ -1179,7 +1144,6 @@ Properties
.. autosummary::
:toctree: api/
- Minute.delta
Minute.freqstr
Minute.kwds
Minute.name
@@ -1194,7 +1158,6 @@ Methods
:toctree: api/
Minute.copy
- Minute.is_anchored
Minute.is_on_offset
Minute.is_month_start
Minute.is_month_end
@@ -1215,7 +1178,6 @@ Properties
.. autosummary::
:toctree: api/
- Second.delta
Second.freqstr
Second.kwds
Second.name
@@ -1230,7 +1192,6 @@ Methods
:toctree: api/
Second.copy
- Second.is_anchored
Second.is_on_offset
Second.is_month_start
Second.is_month_end
@@ -1251,7 +1212,6 @@ Properties
.. autosummary::
:toctree: api/
- Milli.delta
Milli.freqstr
Milli.kwds
Milli.name
@@ -1266,7 +1226,6 @@ Methods
:toctree: api/
Milli.copy
- Milli.is_anchored
Milli.is_on_offset
Milli.is_month_start
Milli.is_month_end
@@ -1287,7 +1246,6 @@ Properties
.. autosummary::
:toctree: api/
- Micro.delta
Micro.freqstr
Micro.kwds
Micro.name
@@ -1302,7 +1260,6 @@ Methods
:toctree: api/
Micro.copy
- Micro.is_anchored
Micro.is_on_offset
Micro.is_month_start
Micro.is_month_end
@@ -1323,7 +1280,6 @@ Properties
.. autosummary::
:toctree: api/
- Nano.delta
Nano.freqstr
Nano.kwds
Nano.name
@@ -1338,7 +1294,6 @@ Methods
:toctree: api/
Nano.copy
- Nano.is_anchored
Nano.is_on_offset
Nano.is_month_start
Nano.is_month_end
diff --git a/doc/source/reference/resampling.rst b/doc/source/reference/resampling.rst
index edbc8090fc849..2e0717081b129 100644
--- a/doc/source/reference/resampling.rst
+++ b/doc/source/reference/resampling.rst
@@ -38,7 +38,6 @@ Upsampling
Resampler.ffill
Resampler.bfill
Resampler.nearest
- Resampler.fillna
Resampler.asfreq
Resampler.interpolate
diff --git a/doc/source/reference/series.rst b/doc/source/reference/series.rst
index af262f9e6c336..43d7480899dc4 100644
--- a/doc/source/reference/series.rst
+++ b/doc/source/reference/series.rst
@@ -47,7 +47,6 @@ Conversion
Series.convert_dtypes
Series.infer_objects
Series.copy
- Series.bool
Series.to_numpy
Series.to_period
Series.to_timestamp
@@ -177,17 +176,16 @@ Reindexing / selection / label manipulation
:toctree: api/
Series.align
+ Series.case_when
Series.drop
Series.droplevel
Series.drop_duplicates
Series.duplicated
Series.equals
- Series.first
Series.head
Series.idxmax
Series.idxmin
Series.isin
- Series.last
Series.reindex
Series.reindex_like
Series.rename
@@ -209,7 +207,6 @@ Missing data handling
.. autosummary::
:toctree: api/
- Series.backfill
Series.bfill
Series.dropna
Series.ffill
@@ -219,7 +216,6 @@ Missing data handling
Series.isnull
Series.notna
Series.notnull
- Series.pad
Series.replace
Reshaping, sorting
@@ -237,10 +233,8 @@ Reshaping, sorting
Series.unstack
Series.explode
Series.searchsorted
- Series.ravel
Series.repeat
Series.squeeze
- Series.view
Combining / comparing / joining / merging
-----------------------------------------
@@ -341,7 +335,6 @@ Datetime properties
Series.dt.tz
Series.dt.freq
Series.dt.unit
- Series.dt.normalize
Datetime methods
^^^^^^^^^^^^^^^^
diff --git a/doc/source/reference/style.rst b/doc/source/reference/style.rst
index 2256876c93e01..0e1d93841d52f 100644
--- a/doc/source/reference/style.rst
+++ b/doc/source/reference/style.rst
@@ -41,6 +41,7 @@ Style application
Styler.map_index
Styler.format
Styler.format_index
+ Styler.format_index_names
Styler.relabel_index
Styler.hide
Styler.concat
diff --git a/doc/source/reference/testing.rst b/doc/source/reference/testing.rst
index a5d61703aceed..1f164d1aa98b4 100644
--- a/doc/source/reference/testing.rst
+++ b/doc/source/reference/testing.rst
@@ -58,8 +58,6 @@ Exceptions and warnings
errors.PossiblePrecisionLoss
errors.PyperclipException
errors.PyperclipWindowsException
- errors.SettingWithCopyError
- errors.SettingWithCopyWarning
errors.SpecificationError
errors.UndefinedVariableError
errors.UnsortedIndexError
diff --git a/doc/source/user_guide/10min.rst b/doc/source/user_guide/10min.rst
index c8e67710c85a9..3cdcb81c14961 100644
--- a/doc/source/user_guide/10min.rst
+++ b/doc/source/user_guide/10min.rst
@@ -19,7 +19,7 @@ Customarily, we import as follows:
Basic data structures in pandas
-------------------------------
-Pandas provides two types of classes for handling data:
+pandas provides two types of classes for handling data:
1. :class:`Series`: a one-dimensional labeled array holding data of any type
such as integers, strings, Python objects etc.
@@ -91,8 +91,8 @@ will be completed:
df2.any df2.combine
df2.append df2.D
df2.apply df2.describe
- df2.applymap df2.diff
df2.B df2.duplicated
+ df2.diff
As you can see, the columns ``A``, ``B``, ``C``, and ``D`` are automatically
tab completed. ``E`` and ``F`` are there as well; the rest of the attributes have been
@@ -563,7 +563,7 @@ columns:
.. ipython:: python
- stacked = df2.stack(future_stack=True)
+ stacked = df2.stack()
stacked
With a "stacked" DataFrame or Series (having a :class:`MultiIndex` as the
diff --git a/doc/source/user_guide/advanced.rst b/doc/source/user_guide/advanced.rst
index 453536098cfbb..f7ab466e92d93 100644
--- a/doc/source/user_guide/advanced.rst
+++ b/doc/source/user_guide/advanced.rst
@@ -11,13 +11,6 @@ and :ref:`other advanced indexing features `.
See the :ref:`Indexing and Selecting Data ` for general indexing documentation.
-.. warning::
-
- Whether a copy or a reference is returned for a setting operation may
- depend on the context. This is sometimes called ``chained assignment`` and
- should be avoided. See :ref:`Returning a View versus Copy
- `.
-
See the :ref:`cookbook` for some advanced strategies.
.. _advanced.hierarchical:
@@ -402,6 +395,7 @@ slicers on a single axis.
Furthermore, you can *set* the values using the following methods.
.. ipython:: python
+ :okwarning:
df2 = dfmi.copy()
df2.loc(axis=0)[:, :, ["C1", "C3"]] = -10
diff --git a/doc/source/user_guide/basics.rst b/doc/source/user_guide/basics.rst
index f7d89110e6c8f..eed3fc149263a 100644
--- a/doc/source/user_guide/basics.rst
+++ b/doc/source/user_guide/basics.rst
@@ -299,8 +299,7 @@ Boolean reductions
~~~~~~~~~~~~~~~~~~
You can apply the reductions: :attr:`~DataFrame.empty`, :meth:`~DataFrame.any`,
-:meth:`~DataFrame.all`, and :meth:`~DataFrame.bool` to provide a
-way to summarize a boolean result.
+:meth:`~DataFrame.all`.
.. ipython:: python
@@ -1309,8 +1308,8 @@ filling method chosen from the following table:
:header: "Method", "Action"
:widths: 30, 50
- pad / ffill, Fill values forward
- bfill / backfill, Fill values backward
+ ffill, Fill values forward
+ bfill, Fill values backward
nearest, Fill from the nearest index value
We illustrate these fill methods on a simple Series:
diff --git a/doc/source/user_guide/categorical.rst b/doc/source/user_guide/categorical.rst
index 8fb991dca02db..7b2fd32303845 100644
--- a/doc/source/user_guide/categorical.rst
+++ b/doc/source/user_guide/categorical.rst
@@ -245,7 +245,8 @@ Equality semantics
Two instances of :class:`~pandas.api.types.CategoricalDtype` compare equal
whenever they have the same categories and order. When comparing two
-unordered categoricals, the order of the ``categories`` is not considered.
+unordered categoricals, the order of the ``categories`` is not considered. Note
+that categories with different dtypes are not the same.
.. ipython:: python
@@ -263,6 +264,16 @@ All instances of ``CategoricalDtype`` compare equal to the string ``'category'``
c1 == "category"
+Notice that the ``categories_dtype`` should be considered, especially when comparing with
+two empty ``CategoricalDtype`` instances.
+
+.. ipython:: python
+
+ c2 = pd.Categorical(np.array([], dtype=object))
+ c3 = pd.Categorical(np.array([], dtype=float))
+
+ c2.dtype == c3.dtype
+
Description
-----------
diff --git a/doc/source/user_guide/cookbook.rst b/doc/source/user_guide/cookbook.rst
index b1a6aa8753be1..3dfc6534f2b64 100644
--- a/doc/source/user_guide/cookbook.rst
+++ b/doc/source/user_guide/cookbook.rst
@@ -311,7 +311,7 @@ The :ref:`multindexing ` docs.
df.columns = pd.MultiIndex.from_tuples([tuple(c.split("_")) for c in df.columns])
df
# Now stack & Reset
- df = df.stack(0, future_stack=True).reset_index(1)
+ df = df.stack(0).reset_index(1)
df
# And fix the labels (Notice the label 'level_1' got added automatically)
df.columns = ["Sample", "All_X", "All_Y"]
@@ -688,7 +688,7 @@ The :ref:`Pivot ` docs.
aggfunc="sum",
margins=True,
)
- table.stack("City", future_stack=True)
+ table.stack("City")
`Frequency table like plyr in R
`__
diff --git a/doc/source/user_guide/copy_on_write.rst b/doc/source/user_guide/copy_on_write.rst
index a083297925007..90353d9f49f00 100644
--- a/doc/source/user_guide/copy_on_write.rst
+++ b/doc/source/user_guide/copy_on_write.rst
@@ -8,16 +8,12 @@ Copy-on-Write (CoW)
.. note::
- Copy-on-Write will become the default in pandas 3.0. We recommend
- :ref:`turning it on now `
- to benefit from all improvements.
+ Copy-on-Write is now the default with pandas 3.0.
Copy-on-Write was first introduced in version 1.5.0. Starting from version 2.0 most of the
optimizations that become possible through CoW are implemented and supported. All possible
optimizations are supported starting from pandas 2.1.
-CoW will be enabled by default in version 3.0.
-
CoW will lead to more predictable behavior since it is not possible to update more than
one object with one statement, e.g. indexing operations or methods won't have side-effects. Additionally, through
delaying copies as long as possible, the average performance and memory usage will improve.
@@ -29,21 +25,25 @@ pandas indexing behavior is tricky to understand. Some operations return views w
other return copies. Depending on the result of the operation, mutating one object
might accidentally mutate another:
-.. ipython:: python
+.. code-block:: ipython
- df = pd.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
- subset = df["foo"]
- subset.iloc[0] = 100
- df
+ In [1]: df = pd.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
+ In [2]: subset = df["foo"]
+ In [3]: subset.iloc[0] = 100
+ In [4]: df
+ Out[4]:
+ foo bar
+ 0 100 4
+ 1 2 5
+ 2 3 6
-Mutating ``subset``, e.g. updating its values, also updates ``df``. The exact behavior is
+
+Mutating ``subset``, e.g. updating its values, also updated ``df``. The exact behavior was
hard to predict. Copy-on-Write solves accidentally modifying more than one object,
-it explicitly disallows this. With CoW enabled, ``df`` is unchanged:
+it explicitly disallows this. ``df`` is unchanged:
.. ipython:: python
- pd.options.mode.copy_on_write = True
-
df = pd.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
subset = df["foo"]
subset.iloc[0] = 100
@@ -57,13 +57,13 @@ applications.
Migrating to Copy-on-Write
--------------------------
-Copy-on-Write will be the default and only mode in pandas 3.0. This means that users
+Copy-on-Write is the default and only mode in pandas 3.0. This means that users
need to migrate their code to be compliant with CoW rules.
-The default mode in pandas will raise warnings for certain cases that will actively
+The default mode in pandas < 3.0 raises warnings for certain cases that will actively
change behavior and thus change user intended behavior.
-We added another mode, e.g.
+pandas 2.2 has a warning mode
.. code-block:: python
@@ -84,7 +84,6 @@ The following few items describe the user visible changes:
**Accessing the underlying array of a pandas object will return a read-only view**
-
.. ipython:: python
ser = pd.Series([1, 2, 3])
@@ -101,16 +100,21 @@ for more details.
**Only one pandas object is updated at once**
-The following code snippet updates both ``df`` and ``subset`` without CoW:
+The following code snippet updated both ``df`` and ``subset`` without CoW:
-.. ipython:: python
+.. code-block:: ipython
- df = pd.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
- subset = df["foo"]
- subset.iloc[0] = 100
- df
+ In [1]: df = pd.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
+ In [2]: subset = df["foo"]
+ In [3]: subset.iloc[0] = 100
+ In [4]: df
+ Out[4]:
+ foo bar
+ 0 100 4
+ 1 2 5
+ 2 3 6
-This won't be possible anymore with CoW, since the CoW rules explicitly forbid this.
+This is not possible anymore with CoW, since the CoW rules explicitly forbid this.
This includes updating a single column as a :class:`Series` and relying on the change
propagating back to the parent :class:`DataFrame`.
This statement can be rewritten into a single statement with ``loc`` or ``iloc`` if
@@ -146,7 +150,7 @@ A different alternative would be to not use ``inplace``:
**Constructors now copy NumPy arrays by default**
-The Series and DataFrame constructors will now copy NumPy array by default when not
+The Series and DataFrame constructors now copies a NumPy array by default when not
otherwise specified. This was changed to avoid mutating a pandas object when the
NumPy array is changed inplace outside of pandas. You can set ``copy=False`` to
avoid this copy.
@@ -162,7 +166,7 @@ that shares data with another DataFrame or Series object inplace.
This avoids side-effects when modifying values and hence, most methods can avoid
actually copying the data and only trigger a copy when necessary.
-The following example will operate inplace with CoW:
+The following example will operate inplace:
.. ipython:: python
@@ -207,15 +211,17 @@ listed in :ref:`Copy-on-Write optimizations `.
Previously, when operating on views, the view and the parent object was modified:
-.. ipython:: python
-
- with pd.option_context("mode.copy_on_write", False):
- df = pd.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
- view = df[:]
- df.iloc[0, 0] = 100
+.. code-block:: ipython
- df
- view
+ In [1]: df = pd.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
+ In [2]: subset = df["foo"]
+ In [3]: subset.iloc[0] = 100
+ In [4]: df
+ Out[4]:
+ foo bar
+ 0 100 4
+ 1 2 5
+ 2 3 6
CoW triggers a copy when ``df`` is changed to avoid mutating ``view`` as well:
@@ -236,16 +242,19 @@ Chained Assignment
Chained assignment references a technique where an object is updated through
two subsequent indexing operations, e.g.
-.. ipython:: python
- :okwarning:
+.. code-block:: ipython
- with pd.option_context("mode.copy_on_write", False):
- df = pd.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
- df["foo"][df["bar"] > 5] = 100
- df
+ In [1]: df = pd.DataFrame({"foo": [1, 2, 3], "bar": [4, 5, 6]})
+ In [2]: df["foo"][df["bar"] > 5] = 100
+ In [3]: df
+ Out[3]:
+ foo bar
+ 0 100 4
+ 1 2 5
+ 2 3 6
-The column ``foo`` is updated where the column ``bar`` is greater than 5.
-This violates the CoW principles though, because it would have to modify the
+The column ``foo`` was updated where the column ``bar`` is greater than 5.
+This violated the CoW principles though, because it would have to modify the
view ``df["foo"]`` and ``df`` in one step. Hence, chained assignment will
consistently never work and raise a ``ChainedAssignmentError`` warning
with CoW enabled:
@@ -272,7 +281,6 @@ shares data with the initial DataFrame:
The array is a copy if the initial DataFrame consists of more than one array:
-
.. ipython:: python
df = pd.DataFrame({"a": [1, 2], "b": [1.5, 2.5]})
@@ -295,7 +303,7 @@ This array is read-only, which means that it can't be modified inplace:
The same holds true for a Series, since a Series always consists of a single array.
-There are two potential solution to this:
+There are two potential solutions to this:
- Trigger a copy manually if you want to avoid updating DataFrames that share memory with your array.
- Make the array writeable. This is a more performant solution but circumvents Copy-on-Write rules, so
@@ -347,22 +355,3 @@ and :meth:`DataFrame.rename`.
These methods return views when Copy-on-Write is enabled, which provides a significant
performance improvement compared to the regular execution.
-
-.. _copy_on_write_enabling:
-
-How to enable CoW
------------------
-
-Copy-on-Write can be enabled through the configuration option ``copy_on_write``. The option can
-be turned on __globally__ through either of the following:
-
-.. ipython:: python
-
- pd.set_option("mode.copy_on_write", True)
-
- pd.options.mode.copy_on_write = True
-
-.. ipython:: python
- :suppress:
-
- pd.options.mode.copy_on_write = False
diff --git a/doc/source/user_guide/dsintro.rst b/doc/source/user_guide/dsintro.rst
index d1e981ee1bbdc..9757a72f13fa8 100644
--- a/doc/source/user_guide/dsintro.rst
+++ b/doc/source/user_guide/dsintro.rst
@@ -41,8 +41,8 @@ Here, ``data`` can be many different things:
* an ndarray
* a scalar value (like 5)
-The passed **index** is a list of axis labels. Thus, this separates into a few
-cases depending on what **data is**:
+The passed **index** is a list of axis labels. The constructor's behavior
+depends on **data**'s type:
**From ndarray**
@@ -97,7 +97,7 @@ provided. The value will be repeated to match the length of **index**.
Series is ndarray-like
~~~~~~~~~~~~~~~~~~~~~~
-:class:`Series` acts very similarly to a ``ndarray`` and is a valid argument to most NumPy functions.
+:class:`Series` acts very similarly to a :class:`numpy.ndarray` and is a valid argument to most NumPy functions.
However, operations such as slicing will also slice the index.
.. ipython:: python
@@ -111,7 +111,7 @@ However, operations such as slicing will also slice the index.
.. note::
We will address array-based indexing like ``s.iloc[[4, 3, 1]]``
- in :ref:`section on indexing `.
+ in the :ref:`section on indexing `.
Like a NumPy array, a pandas :class:`Series` has a single :attr:`~Series.dtype`.
diff --git a/doc/source/user_guide/enhancingperf.rst b/doc/source/user_guide/enhancingperf.rst
index 8c510173819e0..c4721f3a6b09c 100644
--- a/doc/source/user_guide/enhancingperf.rst
+++ b/doc/source/user_guide/enhancingperf.rst
@@ -453,7 +453,7 @@ by evaluate arithmetic and boolean expression all at once for large :class:`~pan
:func:`~pandas.eval` is many orders of magnitude slower for
smaller expressions or objects than plain Python. A good rule of thumb is
to only use :func:`~pandas.eval` when you have a
- :class:`.DataFrame` with more than 10,000 rows.
+ :class:`~pandas.core.frame.DataFrame` with more than 10,000 rows.
Supported syntax
~~~~~~~~~~~~~~~~
diff --git a/doc/source/user_guide/groupby.rst b/doc/source/user_guide/groupby.rst
index 11863f8aead31..7f957a8b16787 100644
--- a/doc/source/user_guide/groupby.rst
+++ b/doc/source/user_guide/groupby.rst
@@ -137,15 +137,6 @@ We could naturally group by either the ``A`` or ``B`` columns, or both:
``df.groupby('A')`` is just syntactic sugar for ``df.groupby(df['A'])``.
-If we also have a MultiIndex on columns ``A`` and ``B``, we can group by all
-the columns except the one we specify:
-
-.. ipython:: python
-
- df2 = df.set_index(["A", "B"])
- grouped = df2.groupby(level=df2.index.names.difference(["B"]))
- grouped.sum()
-
The above GroupBy will split the DataFrame on its index (rows). To split by columns, first do
a transpose:
@@ -247,7 +238,7 @@ GroupBy object attributes
~~~~~~~~~~~~~~~~~~~~~~~~~
The ``groups`` attribute is a dictionary whose keys are the computed unique groups
-and corresponding values are the axis labels belonging to each group. In the
+and corresponding values are the index labels belonging to each group. In the
above example we have:
.. ipython:: python
@@ -289,7 +280,7 @@ the number of groups, which is the same as the length of the ``groups`` dictiona
In [1]: gb. # noqa: E225, E999
gb.agg gb.boxplot gb.cummin gb.describe gb.filter gb.get_group gb.height gb.last gb.median gb.ngroups gb.plot gb.rank gb.std gb.transform
gb.aggregate gb.count gb.cumprod gb.dtype gb.first gb.groups gb.hist gb.max gb.min gb.nth gb.prod gb.resample gb.sum gb.var
- gb.apply gb.cummax gb.cumsum gb.fillna gb.gender gb.head gb.indices gb.mean gb.name gb.ohlc gb.quantile gb.size gb.tail gb.weight
+ gb.apply gb.cummax gb.cumsum gb.gender gb.head gb.indices gb.mean gb.name gb.ohlc gb.quantile gb.size gb.tail gb.weight
.. _groupby.multiindex:
@@ -1730,4 +1721,4 @@ column index name will be used as the name of the inserted column:
result
- result.stack(future_stack=True)
+ result.stack()
diff --git a/doc/source/user_guide/indexing.rst b/doc/source/user_guide/indexing.rst
index 4954ee1538697..24cdbad41fe60 100644
--- a/doc/source/user_guide/indexing.rst
+++ b/doc/source/user_guide/indexing.rst
@@ -29,13 +29,6 @@ this area.
production code, we recommended that you take advantage of the optimized
pandas data access methods exposed in this chapter.
-.. warning::
-
- Whether a copy or a reference is returned for a setting operation, may
- depend on the context. This is sometimes called ``chained assignment`` and
- should be avoided. See :ref:`Returning a View versus Copy
- `.
-
See the :ref:`MultiIndex / Advanced Indexing ` for ``MultiIndex`` and more advanced indexing documentation.
See the :ref:`cookbook` for some advanced strategies.
@@ -299,12 +292,6 @@ largely as a convenience since it is such a common operation.
Selection by label
------------------
-.. warning::
-
- Whether a copy or a reference is returned for a setting operation, may depend on the context.
- This is sometimes called ``chained assignment`` and should be avoided.
- See :ref:`Returning a View versus Copy `.
-
.. warning::
``.loc`` is strict when you present slicers that are not compatible (or convertible) with the index type. For example
@@ -445,12 +432,6 @@ For more information about duplicate labels, see
Selection by position
---------------------
-.. warning::
-
- Whether a copy or a reference is returned for a setting operation, may depend on the context.
- This is sometimes called ``chained assignment`` and should be avoided.
- See :ref:`Returning a View versus Copy `.
-
pandas provides a suite of methods in order to get **purely integer based indexing**. The semantics follow closely Python and NumPy slicing. These are ``0-based`` indexing. When slicing, the start bound is *included*, while the upper bound is *excluded*. Trying to use a non-integer, even a **valid** label will raise an ``IndexError``.
The ``.iloc`` attribute is the primary access method. The following are valid inputs:
@@ -1722,234 +1703,10 @@ You can assign a custom index to the ``index`` attribute:
df_idx.index = pd.Index([10, 20, 30, 40], name="a")
df_idx
-.. _indexing.view_versus_copy:
-
-Returning a view versus a copy
-------------------------------
-
-.. warning::
-
- :ref:`Copy-on-Write `
- will become the new default in pandas 3.0. This means than chained indexing will
- never work. As a consequence, the ``SettingWithCopyWarning`` won't be necessary
- anymore.
- See :ref:`this section `
- for more context.
- We recommend turning Copy-on-Write on to leverage the improvements with
-
- ```
- pd.options.mode.copy_on_write = True
- ```
-
- even before pandas 3.0 is available.
-
-When setting values in a pandas object, care must be taken to avoid what is called
-``chained indexing``. Here is an example.
-
-.. ipython:: python
-
- dfmi = pd.DataFrame([list('abcd'),
- list('efgh'),
- list('ijkl'),
- list('mnop')],
- columns=pd.MultiIndex.from_product([['one', 'two'],
- ['first', 'second']]))
- dfmi
-
-Compare these two access methods:
-
-.. ipython:: python
-
- dfmi['one']['second']
-
-.. ipython:: python
-
- dfmi.loc[:, ('one', 'second')]
-
-These both yield the same results, so which should you use? It is instructive to understand the order
-of operations on these and why method 2 (``.loc``) is much preferred over method 1 (chained ``[]``).
-
-``dfmi['one']`` selects the first level of the columns and returns a DataFrame that is singly-indexed.
-Then another Python operation ``dfmi_with_one['second']`` selects the series indexed by ``'second'``.
-This is indicated by the variable ``dfmi_with_one`` because pandas sees these operations as separate events.
-e.g. separate calls to ``__getitem__``, so it has to treat them as linear operations, they happen one after another.
-
-Contrast this to ``df.loc[:,('one','second')]`` which passes a nested tuple of ``(slice(None),('one','second'))`` to a single call to
-``__getitem__``. This allows pandas to deal with this as a single entity. Furthermore this order of operations *can* be significantly
-faster, and allows one to index *both* axes if so desired.
-
Why does assignment fail when using chained indexing?
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-.. warning::
-
- :ref:`Copy-on-Write `
- will become the new default in pandas 3.0. This means than chained indexing will
- never work. As a consequence, the ``SettingWithCopyWarning`` won't be necessary
- anymore.
- See :ref:`this section `
- for more context.
- We recommend turning Copy-on-Write on to leverage the improvements with
-
- ```
- pd.options.mode.copy_on_write = True
- ```
-
- even before pandas 3.0 is available.
-
-The problem in the previous section is just a performance issue. What's up with
-the ``SettingWithCopy`` warning? We don't **usually** throw warnings around when
-you do something that might cost a few extra milliseconds!
-
-But it turns out that assigning to the product of chained indexing has
-inherently unpredictable results. To see this, think about how the Python
-interpreter executes this code:
-
-.. code-block:: python
-
- dfmi.loc[:, ('one', 'second')] = value
- # becomes
- dfmi.loc.__setitem__((slice(None), ('one', 'second')), value)
-
-But this code is handled differently:
-
-.. code-block:: python
-
- dfmi['one']['second'] = value
- # becomes
- dfmi.__getitem__('one').__setitem__('second', value)
-
-See that ``__getitem__`` in there? Outside of simple cases, it's very hard to
-predict whether it will return a view or a copy (it depends on the memory layout
-of the array, about which pandas makes no guarantees), and therefore whether
-the ``__setitem__`` will modify ``dfmi`` or a temporary object that gets thrown
-out immediately afterward. **That's** what ``SettingWithCopy`` is warning you
-about!
-
-.. note:: You may be wondering whether we should be concerned about the ``loc``
- property in the first example. But ``dfmi.loc`` is guaranteed to be ``dfmi``
- itself with modified indexing behavior, so ``dfmi.loc.__getitem__`` /
- ``dfmi.loc.__setitem__`` operate on ``dfmi`` directly. Of course,
- ``dfmi.loc.__getitem__(idx)`` may be a view or a copy of ``dfmi``.
-
-Sometimes a ``SettingWithCopy`` warning will arise at times when there's no
-obvious chained indexing going on. **These** are the bugs that
-``SettingWithCopy`` is designed to catch! pandas is probably trying to warn you
-that you've done this:
-
-.. code-block:: python
-
- def do_something(df):
- foo = df[['bar', 'baz']] # Is foo a view? A copy? Nobody knows!
- # ... many lines here ...
- # We don't know whether this will modify df or not!
- foo['quux'] = value
- return foo
-
-Yikes!
-
-.. _indexing.evaluation_order:
-
-Evaluation order matters
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. warning::
-
- :ref:`Copy-on-Write `
- will become the new default in pandas 3.0. This means than chained indexing will
- never work. As a consequence, the ``SettingWithCopyWarning`` won't be necessary
- anymore.
- See :ref:`this section `
- for more context.
- We recommend turning Copy-on-Write on to leverage the improvements with
-
- ```
- pd.options.mode.copy_on_write = True
- ```
-
- even before pandas 3.0 is available.
-
-When you use chained indexing, the order and type of the indexing operation
-partially determine whether the result is a slice into the original object, or
-a copy of the slice.
-
-pandas has the ``SettingWithCopyWarning`` because assigning to a copy of a
-slice is frequently not intentional, but a mistake caused by chained indexing
-returning a copy where a slice was expected.
-
-If you would like pandas to be more or less trusting about assignment to a
-chained indexing expression, you can set the :ref:`option `
-``mode.chained_assignment`` to one of these values:
-
-* ``'warn'``, the default, means a ``SettingWithCopyWarning`` is printed.
-* ``'raise'`` means pandas will raise a ``SettingWithCopyError``
- you have to deal with.
-* ``None`` will suppress the warnings entirely.
-
-.. ipython:: python
- :okwarning:
-
- dfb = pd.DataFrame({'a': ['one', 'one', 'two',
- 'three', 'two', 'one', 'six'],
- 'c': np.arange(7)})
-
- # This will show the SettingWithCopyWarning
- # but the frame values will be set
- dfb['c'][dfb['a'].str.startswith('o')] = 42
-
-This however is operating on a copy and will not work.
-
-.. ipython:: python
- :okwarning:
- :okexcept:
-
- with pd.option_context('mode.chained_assignment','warn'):
- dfb[dfb['a'].str.startswith('o')]['c'] = 42
-
-A chained assignment can also crop up in setting in a mixed dtype frame.
-
-.. note::
-
- These setting rules apply to all of ``.loc/.iloc``.
-
-The following is the recommended access method using ``.loc`` for multiple items (using ``mask``) and a single item using a fixed index:
-
-.. ipython:: python
-
- dfc = pd.DataFrame({'a': ['one', 'one', 'two',
- 'three', 'two', 'one', 'six'],
- 'c': np.arange(7)})
- dfd = dfc.copy()
- # Setting multiple items using a mask
- mask = dfd['a'].str.startswith('o')
- dfd.loc[mask, 'c'] = 42
- dfd
-
- # Setting a single item
- dfd = dfc.copy()
- dfd.loc[2, 'a'] = 11
- dfd
-
-The following *can* work at times, but it is not guaranteed to, and therefore should be avoided:
-
-.. ipython:: python
- :okwarning:
-
- dfd = dfc.copy()
- dfd['a'][2] = 111
- dfd
-
-Last, the subsequent example will **not** work at all, and so should be avoided:
-
-.. ipython:: python
- :okwarning:
- :okexcept:
-
- with pd.option_context('mode.chained_assignment','raise'):
- dfd.loc[0]['a'] = 1111
-
-.. warning::
-
- The chained assignment warnings / exceptions are aiming to inform the user of a possibly invalid
- assignment. There may be false positives; situations where a chained assignment is inadvertently
- reported.
+:ref:`Copy-on-Write ` is the new default with pandas 3.0.
+This means than chained indexing will never work.
+See :ref:`this section `
+for more context.
diff --git a/doc/source/user_guide/io.rst b/doc/source/user_guide/io.rst
index 6148086452d54..db2326d5b9754 100644
--- a/doc/source/user_guide/io.rst
+++ b/doc/source/user_guide/io.rst
@@ -36,7 +36,6 @@ The pandas I/O API is a set of top level ``reader`` functions accessed like
binary;`SPSS `__;:ref:`read_spss`;
binary;`Python Pickle Format `__;:ref:`read_pickle`;:ref:`to_pickle`
SQL;`SQL `__;:ref:`read_sql`;:ref:`to_sql`
- SQL;`Google BigQuery `__;:ref:`read_gbq`;:ref:`to_gbq`
:ref:`Here ` is an informal performance comparison for some of these IO methods.
@@ -61,8 +60,8 @@ Basic
+++++
filepath_or_buffer : various
- Either a path to a file (a :class:`python:str`, :class:`python:pathlib.Path`,
- or :class:`py:py._path.local.LocalPath`), URL (including http, ftp, and S3
+ Either a path to a file (a :class:`python:str`, :class:`python:pathlib.Path`)
+ URL (including http, ftp, and S3
locations), or any object with a ``read()`` method (such as an open file or
:class:`~python:io.StringIO`).
sep : str, defaults to ``','`` for :func:`read_csv`, ``\t`` for :func:`read_table`
@@ -1045,7 +1044,7 @@ Writing CSVs to binary file objects
``df.to_csv(..., mode="wb")`` allows writing a CSV to a file object
opened binary mode. In most cases, it is not necessary to specify
-``mode`` as Pandas will auto-detect whether the file object is
+``mode`` as pandas will auto-detect whether the file object is
opened in text or binary mode.
.. ipython:: python
@@ -1605,7 +1604,7 @@ Specifying ``iterator=True`` will also return the ``TextFileReader`` object:
Specifying the parser engine
''''''''''''''''''''''''''''
-Pandas currently supports three engines, the C engine, the python engine, and an experimental
+pandas currently supports three engines, the C engine, the python engine, and an experimental
pyarrow engine (requires the ``pyarrow`` package). In general, the pyarrow engine is fastest
on larger workloads and is equivalent in speed to the C engine on most other workloads.
The python engine tends to be slower than the pyarrow and C engines on most workloads. However,
@@ -1704,7 +1703,7 @@ option parameter:
.. code-block:: python
- storage_options = {"client_kwargs": {"endpoint_url": "http://127.0.0.1:5555"}}}
+ storage_options = {"client_kwargs": {"endpoint_url": "http://127.0.0.1:5555"}}
df = pd.read_json("s3://pandas-test/test-1", storage_options=storage_options)
More sample configurations and documentation can be found at `S3Fs documentation
@@ -3015,14 +3014,15 @@ Read in the content of the "books.xml" as instance of ``StringIO`` or
Even read XML from AWS S3 buckets such as NIH NCBI PMC Article Datasets providing
Biomedical and Life Science Jorurnals:
-.. ipython:: python
- :okwarning:
+.. code-block:: python
- df = pd.read_xml(
- "s3://pmc-oa-opendata/oa_comm/xml/all/PMC1236943.xml",
- xpath=".//journal-meta",
- )
- df
+ >>> df = pd.read_xml(
+ ... "s3://pmc-oa-opendata/oa_comm/xml/all/PMC1236943.xml",
+ ... xpath=".//journal-meta",
+ ...)
+ >>> df
+ journal-id journal-title issn publisher
+ 0 Cardiovasc Ultrasound Cardiovascular Ultrasound 1476-7120 NaN
With `lxml`_ as default ``parser``, you access the full-featured XML library
that extends Python's ElementTree API. One powerful tool is ability to query
@@ -3247,7 +3247,7 @@ output (as shown below for demonstration) for easier parse into ``DataFrame``:
"""
- df = pd.read_xml(StringIO(xml), stylesheet=xsl)
+ df = pd.read_xml(StringIO(xml), stylesheet=StringIO(xsl))
df
For very large XML files that can range in hundreds of megabytes to gigabytes, :func:`pandas.read_xml`
@@ -3418,7 +3418,7 @@ Write an XML and transform with stylesheet:
"""
- print(geom_df.to_xml(stylesheet=xsl))
+ print(geom_df.to_xml(stylesheet=StringIO(xsl)))
XML Final Notes
@@ -3471,20 +3471,15 @@ saving a ``DataFrame`` to Excel. Generally the semantics are
similar to working with :ref:`csv` data.
See the :ref:`cookbook` for some advanced strategies.
-.. warning::
-
- The `xlrd `__.
+
+
+.. code-block:: python
+
+ css = "border: 1px solid black; font-weight: bold;"
+ df.style.map_index(lambda x: css).map_index(lambda x: css, axis=1).to_excel("myfile.xlsx")
+
Using the `Xlsxwriter`_ engine provides many options for controlling the
format of an Excel worksheet created with the ``to_excel`` method. Excellent examples can be found in the
`Xlsxwriter`_ documentation here: https://xlsxwriter.readthedocs.io/working_with_pandas.html
@@ -6100,10 +6109,6 @@ Google BigQuery
The ``pandas-gbq`` package provides functionality to read/write from Google BigQuery.
-pandas integrates with this external package. if ``pandas-gbq`` is installed, you can
-use the pandas methods ``pd.read_gbq`` and ``DataFrame.to_gbq``, which will call the
-respective functions from ``pandas-gbq``.
-
Full documentation can be found `here
+
RangeIndex: 1000000 entries, 0 to 999999
Data columns (total 2 columns):
A 1000000 non-null float64
diff --git a/doc/source/user_guide/merging.rst b/doc/source/user_guide/merging.rst
index c9c8478a719f0..1edf3908936db 100644
--- a/doc/source/user_guide/merging.rst
+++ b/doc/source/user_guide/merging.rst
@@ -249,7 +249,7 @@ a :class:`MultiIndex`) associate specific keys with each original :class:`DataFr
p.plot(frames, result, labels=["df1", "df2", "df3"], vertical=True)
plt.close("all");
-The ``keys`` argument cane override the column names
+The ``keys`` argument can override the column names
when creating a new :class:`DataFrame` based on existing :class:`Series`.
.. ipython:: python
diff --git a/doc/source/user_guide/missing_data.rst b/doc/source/user_guide/missing_data.rst
index 4a2aa565dd15c..2e104ac06f9f4 100644
--- a/doc/source/user_guide/missing_data.rst
+++ b/doc/source/user_guide/missing_data.rst
@@ -21,7 +21,7 @@ is that the original data type will be coerced to ``np.float64`` or ``object``.
pd.Series([True, False], dtype=np.bool_).reindex([0, 1, 2])
:class:`NaT` for NumPy ``np.datetime64``, ``np.timedelta64``, and :class:`PeriodDtype`. For typing applications,
-use :class:`api.types.NaTType`.
+use :class:`api.typing.NaTType`.
.. ipython:: python
@@ -30,7 +30,7 @@ use :class:`api.types.NaTType`.
pd.Series(["2020", "2020"], dtype=pd.PeriodDtype("D")).reindex([0, 1, 2])
:class:`NA` for :class:`StringDtype`, :class:`Int64Dtype` (and other bit widths),
-:class:`Float64Dtype`(and other bit widths), :class:`BooleanDtype` and :class:`ArrowDtype`.
+:class:`Float64Dtype` (and other bit widths), :class:`BooleanDtype` and :class:`ArrowDtype`.
These types will maintain the original data type of the data.
For typing applications, use :class:`api.types.NAType`.
@@ -88,7 +88,7 @@ To detect these missing value, use the :func:`isna` or :func:`notna` methods.
.. warning::
- Experimental: the behaviour of :class:`NA`` can still change without warning.
+ Experimental: the behaviour of :class:`NA` can still change without warning.
Starting from pandas 1.0, an experimental :class:`NA` value (singleton) is
available to represent scalar missing values. The goal of :class:`NA` is provide a
@@ -105,7 +105,7 @@ dtype, it will use :class:`NA`:
s[2]
s[2] is pd.NA
-Currently, pandas does not yet use those data types using :class:`NA` by default
+Currently, pandas does not use those data types using :class:`NA` by default in
a :class:`DataFrame` or :class:`Series`, so you need to specify
the dtype explicitly. An easy way to convert to those dtypes is explained in the
:ref:`conversion section `.
@@ -253,8 +253,8 @@ Conversion
^^^^^^^^^^
If you have a :class:`DataFrame` or :class:`Series` using ``np.nan``,
-:meth:`Series.convert_dtypes` and :meth:`DataFrame.convert_dtypes`
-in :class:`DataFrame` that can convert data to use the data types that use :class:`NA`
+:meth:`DataFrame.convert_dtypes` and :meth:`Series.convert_dtypes`, respectively,
+will convert your data to use the nullable data types supporting :class:`NA`,
such as :class:`Int64Dtype` or :class:`ArrowDtype`. This is especially helpful after reading
in data sets from IO methods where data types were inferred.
diff --git a/doc/source/user_guide/reshaping.rst b/doc/source/user_guide/reshaping.rst
index a35cfb396f1f6..3347f3a2534f4 100644
--- a/doc/source/user_guide/reshaping.rst
+++ b/doc/source/user_guide/reshaping.rst
@@ -163,7 +163,7 @@ as having a multi-level index:
.. ipython:: python
- table.stack(future_stack=True)
+ table.stack()
.. _reshaping.stacking:
@@ -209,7 +209,7 @@ stacked level becomes the new lowest level in a :class:`MultiIndex` on the colum
.. ipython:: python
- stacked = df2.stack(future_stack=True)
+ stacked = df2.stack()
stacked
With a "stacked" :class:`DataFrame` or :class:`Series` (having a :class:`MultiIndex` as the
@@ -245,7 +245,7 @@ will result in a **sorted** copy of the original :class:`DataFrame` or :class:`S
index = pd.MultiIndex.from_product([[2, 1], ["a", "b"]])
df = pd.DataFrame(np.random.randn(4), index=index, columns=["A"])
df
- all(df.unstack().stack(future_stack=True) == df.sort_index())
+ all(df.unstack().stack() == df.sort_index())
.. _reshaping.stack_multiple:
@@ -270,16 +270,16 @@ processed individually.
df = pd.DataFrame(np.random.randn(4, 4), columns=columns)
df
- df.stack(level=["animal", "hair_length"], future_stack=True)
+ df.stack(level=["animal", "hair_length"])
The list of levels can contain either level names or level numbers but
not a mixture of the two.
.. ipython:: python
- # df.stack(level=['animal', 'hair_length'], future_stack=True)
+ # df.stack(level=['animal', 'hair_length'])
# from above is equivalent to:
- df.stack(level=[1, 2], future_stack=True)
+ df.stack(level=[1, 2])
Missing data
~~~~~~~~~~~~
diff --git a/doc/source/user_guide/scale.rst b/doc/source/user_guide/scale.rst
index b262de5d71439..29df2994fbc35 100644
--- a/doc/source/user_guide/scale.rst
+++ b/doc/source/user_guide/scale.rst
@@ -156,7 +156,7 @@ fits in memory, you can work with datasets that are much larger than memory.
Chunking works well when the operation you're performing requires zero or minimal
coordination between chunks. For more complicated workflows, you're better off
- :ref:`using another library `.
+ :ref:`using other libraries `.
Suppose we have an even larger "logical dataset" on disk that's a directory of parquet
files. Each file in the directory represents a different year of the entire dataset.
@@ -219,160 +219,10 @@ different library that implements these out-of-core algorithms for you.
.. _scale.other_libraries:
-Use Dask
---------
+Use Other Libraries
+-------------------
-pandas is just one library offering a DataFrame API. Because of its popularity,
-pandas' API has become something of a standard that other libraries implement.
-The pandas documentation maintains a list of libraries implementing a DataFrame API
-in `the ecosystem page `_.
-
-For example, `Dask`_, a parallel computing library, has `dask.dataframe`_, a
-pandas-like API for working with larger than memory datasets in parallel. Dask
-can use multiple threads or processes on a single machine, or a cluster of
-machines to process data in parallel.
-
-
-We'll import ``dask.dataframe`` and notice that the API feels similar to pandas.
-We can use Dask's ``read_parquet`` function, but provide a globstring of files to read in.
-
-.. ipython:: python
- :okwarning:
-
- import dask.dataframe as dd
-
- ddf = dd.read_parquet("data/timeseries/ts*.parquet", engine="pyarrow")
- ddf
-
-Inspecting the ``ddf`` object, we see a few things
-
-* There are familiar attributes like ``.columns`` and ``.dtypes``
-* There are familiar methods like ``.groupby``, ``.sum``, etc.
-* There are new attributes like ``.npartitions`` and ``.divisions``
-
-The partitions and divisions are how Dask parallelizes computation. A **Dask**
-DataFrame is made up of many pandas :class:`pandas.DataFrame`. A single method call on a
-Dask DataFrame ends up making many pandas method calls, and Dask knows how to
-coordinate everything to get the result.
-
-.. ipython:: python
-
- ddf.columns
- ddf.dtypes
- ddf.npartitions
-
-One major difference: the ``dask.dataframe`` API is *lazy*. If you look at the
-repr above, you'll notice that the values aren't actually printed out; just the
-column names and dtypes. That's because Dask hasn't actually read the data yet.
-Rather than executing immediately, doing operations build up a **task graph**.
-
-.. ipython:: python
- :okwarning:
-
- ddf
- ddf["name"]
- ddf["name"].value_counts()
-
-Each of these calls is instant because the result isn't being computed yet.
-We're just building up a list of computation to do when someone needs the
-result. Dask knows that the return type of a :class:`pandas.Series.value_counts`
-is a pandas :class:`pandas.Series` with a certain dtype and a certain name. So the Dask version
-returns a Dask Series with the same dtype and the same name.
-
-To get the actual result you can call ``.compute()``.
-
-.. ipython:: python
- :okwarning:
-
- %time ddf["name"].value_counts().compute()
-
-At that point, you get back the same thing you'd get with pandas, in this case
-a concrete pandas :class:`pandas.Series` with the count of each ``name``.
-
-Calling ``.compute`` causes the full task graph to be executed. This includes
-reading the data, selecting the columns, and doing the ``value_counts``. The
-execution is done *in parallel* where possible, and Dask tries to keep the
-overall memory footprint small. You can work with datasets that are much larger
-than memory, as long as each partition (a regular pandas :class:`pandas.DataFrame`) fits in memory.
-
-By default, ``dask.dataframe`` operations use a threadpool to do operations in
-parallel. We can also connect to a cluster to distribute the work on many
-machines. In this case we'll connect to a local "cluster" made up of several
-processes on this single machine.
-
-.. code-block:: python
-
- >>> from dask.distributed import Client, LocalCluster
-
- >>> cluster = LocalCluster()
- >>> client = Client(cluster)
- >>> client
-
-
-Once this ``client`` is created, all of Dask's computation will take place on
-the cluster (which is just processes in this case).
-
-Dask implements the most used parts of the pandas API. For example, we can do
-a familiar groupby aggregation.
-
-.. ipython:: python
- :okwarning:
-
- %time ddf.groupby("name")[["x", "y"]].mean().compute().head()
-
-The grouping and aggregation is done out-of-core and in parallel.
-
-When Dask knows the ``divisions`` of a dataset, certain optimizations are
-possible. When reading parquet datasets written by dask, the divisions will be
-known automatically. In this case, since we created the parquet files manually,
-we need to supply the divisions manually.
-
-.. ipython:: python
- :okwarning:
-
- N = 12
- starts = [f"20{i:>02d}-01-01" for i in range(N)]
- ends = [f"20{i:>02d}-12-13" for i in range(N)]
-
- divisions = tuple(pd.to_datetime(starts)) + (pd.Timestamp(ends[-1]),)
- ddf.divisions = divisions
- ddf
-
-Now we can do things like fast random access with ``.loc``.
-
-.. ipython:: python
- :okwarning:
-
- ddf.loc["2002-01-01 12:01":"2002-01-01 12:05"].compute()
-
-Dask knows to just look in the 3rd partition for selecting values in 2002. It
-doesn't need to look at any other data.
-
-Many workflows involve a large amount of data and processing it in a way that
-reduces the size to something that fits in memory. In this case, we'll resample
-to daily frequency and take the mean. Once we've taken the mean, we know the
-results will fit in memory, so we can safely call ``compute`` without running
-out of memory. At that point it's just a regular pandas object.
-
-.. ipython:: python
- :okwarning:
-
- @savefig dask_resample.png
- ddf[["x", "y"]].resample("1D").mean().cumsum().compute().plot()
-
-.. ipython:: python
- :suppress:
-
- import shutil
-
- shutil.rmtree("data/timeseries")
-
-These Dask examples have all be done using multiple processes on a single
-machine. Dask can be `deployed on a cluster
-`_ to scale up to even larger
-datasets.
-
-You see more dask examples at https://examples.dask.org.
-
-.. _Dask: https://dask.org
-.. _dask.dataframe: https://docs.dask.org/en/latest/dataframe.html
+There are other libraries which provide similar APIs to pandas and work nicely with pandas DataFrame,
+and can give you the ability to scale your large dataset processing and analytics
+by parallel runtime, distributed memory, clustering, etc. You can find more information
+in `the ecosystem page `_.
diff --git a/doc/source/user_guide/style.ipynb b/doc/source/user_guide/style.ipynb
index f22a506499cf4..f831723f44931 100644
--- a/doc/source/user_guide/style.ipynb
+++ b/doc/source/user_guide/style.ipynb
@@ -1280,7 +1280,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "You can create \"heatmaps\" with the `background_gradient` and `text_gradient` methods. These require matplotlib, and we'll use [Seaborn](http://seaborn.pydata.org/) to get a nice colormap."
+ "You can create \"heatmaps\" with the `background_gradient` and `text_gradient` methods. These require matplotlib, and we'll use [Seaborn](https://seaborn.pydata.org/) to get a nice colormap."
]
},
{
@@ -1621,7 +1621,9 @@
"source": [
"### HTML Escaping\n",
"\n",
- "Suppose you have to display HTML within HTML, that can be a bit of pain when the renderer can't distinguish. You can use the `escape` formatting option to handle this, and even use it within a formatter that contains HTML itself."
+ "Suppose you have to display HTML within HTML, that can be a bit of pain when the renderer can't distinguish. You can use the `escape` formatting option to handle this, and even use it within a formatter that contains HTML itself.\n",
+ "\n",
+ "Note that if you're using `Styler` on untrusted, user-provided input to serve HTML then you should escape the input to prevent security vulnerabilities. See the Jinja2 documentation for more."
]
},
{
diff --git a/doc/source/user_guide/timeseries.rst b/doc/source/user_guide/timeseries.rst
index 0f0e6271d8329..37413722de96f 100644
--- a/doc/source/user_guide/timeseries.rst
+++ b/doc/source/user_guide/timeseries.rst
@@ -797,8 +797,6 @@ There are several time/date properties that one can access from ``Timestamp`` or
timetz,"Returns datetime.time as local time with timezone information"
dayofyear,"The ordinal day of year"
day_of_year,"The ordinal day of year"
- weekofyear,"The week ordinal of the year"
- week,"The week ordinal of the year"
dayofweek,"The number of the day of the week with Monday=0, Sunday=6"
day_of_week,"The number of the day of the week with Monday=0, Sunday=6"
weekday,"The number of the day of the week with Monday=0, Sunday=6"
@@ -812,6 +810,10 @@ There are several time/date properties that one can access from ``Timestamp`` or
is_year_end,"Logical indicating if last day of year (defined by frequency)"
is_leap_year,"Logical indicating if the date belongs to a leap year"
+.. note::
+
+ You can use ``DatetimeIndex.isocalendar().week`` to access week of year date information.
+
Furthermore, if you have a ``Series`` with datetimelike values, then you can
access these properties via the ``.dt`` accessor, as detailed in the section
on :ref:`.dt accessors`.
@@ -1327,8 +1329,8 @@ frequencies. We will refer to these aliases as *period aliases*.
.. deprecated:: 2.2.0
- Aliases ``A``, ``H``, ``T``, ``S``, ``L``, ``U``, and ``N`` are deprecated in favour of the aliases
- ``Y``, ``h``, ``min``, ``s``, ``ms``, ``us``, and ``ns``.
+ Aliases ``H``, ``T``, ``S``, ``L``, ``U``, and ``N`` are deprecated in favour of the aliases
+ ``h``, ``min``, ``s``, ``ms``, ``us``, and ``ns``.
Combining aliases
@@ -1466,11 +1468,16 @@ or some other non-observed day. Defined observance rules are:
:header: "Rule", "Description"
:widths: 15, 70
+ "next_workday", "move Saturday and Sunday to Monday"
+ "previous_workday", "move Saturday and Sunday to Friday"
"nearest_workday", "move Saturday to Friday and Sunday to Monday"
+ "before_nearest_workday", "apply ``nearest_workday`` and then move to previous workday before that day"
+ "after_nearest_workday", "apply ``nearest_workday`` and then move to next workday after that day"
"sunday_to_monday", "move Sunday to following Monday"
"next_monday_or_tuesday", "move Saturday to Monday and Sunday/Monday to Tuesday"
"previous_friday", move Saturday and Sunday to previous Friday"
"next_monday", "move Saturday and Sunday to following Monday"
+ "weekend_to_monday", "same as ``next_monday``"
An example of how holidays and holiday calendars are defined:
diff --git a/doc/source/user_guide/window.rst b/doc/source/user_guide/window.rst
index d997aa119b359..e25c4c2441920 100644
--- a/doc/source/user_guide/window.rst
+++ b/doc/source/user_guide/window.rst
@@ -79,9 +79,9 @@ which will first group the data by the specified keys and then perform a windowi
.. versionadded:: 1.3.0
Some windowing operations also support the ``method='table'`` option in the constructor which
-performs the windowing operation over an entire :class:`DataFrame` instead of a single column or row at a time.
-This can provide a useful performance benefit for a :class:`DataFrame` with many columns or rows
-(with the corresponding ``axis`` argument) or the ability to utilize other columns during the windowing
+performs the windowing operation over an entire :class:`DataFrame` instead of a single column at a time.
+This can provide a useful performance benefit for a :class:`DataFrame` with many columns
+or the ability to utilize other columns during the windowing
operation. The ``method='table'`` option can only be used if ``engine='numba'`` is specified
in the corresponding method call.
diff --git a/doc/source/whatsnew/index.rst b/doc/source/whatsnew/index.rst
index 09d1ae08df57a..1a1ecdd0effee 100644
--- a/doc/source/whatsnew/index.rst
+++ b/doc/source/whatsnew/index.rst
@@ -10,13 +10,13 @@ This is the list of changes to pandas between each release. For full details,
see the `commit logs `, a collection
diff --git a/doc/source/whatsnew/v0.12.0.rst b/doc/source/whatsnew/v0.12.0.rst
index 59d104cb3e96c..c805758f85b35 100644
--- a/doc/source/whatsnew/v0.12.0.rst
+++ b/doc/source/whatsnew/v0.12.0.rst
@@ -201,12 +201,12 @@ IO enhancements
You can use ``pd.read_html()`` to read the output from ``DataFrame.to_html()`` like so
.. ipython:: python
- :okwarning:
+ import io
df = pd.DataFrame({"a": range(3), "b": list("abc")})
print(df)
html = df.to_html()
- alist = pd.read_html(html, index_col=0)
+ alist = pd.read_html(io.StringIO(html), index_col=0)
print(df == alist[0])
Note that ``alist`` here is a Python ``list`` so ``pd.read_html()`` and
diff --git a/doc/source/whatsnew/v0.13.0.rst b/doc/source/whatsnew/v0.13.0.rst
index f2e29121760ab..a624e81d17db9 100644
--- a/doc/source/whatsnew/v0.13.0.rst
+++ b/doc/source/whatsnew/v0.13.0.rst
@@ -172,7 +172,7 @@ API changes
statistical mode(s) by axis/Series. (:issue:`5367`)
- Chained assignment will now by default warn if the user is assigning to a copy. This can be changed
- with the option ``mode.chained_assignment``, allowed options are ``raise/warn/None``. See :ref:`the docs`.
+ with the option ``mode.chained_assignment``, allowed options are ``raise/warn/None``.
.. ipython:: python
diff --git a/doc/source/whatsnew/v0.13.1.rst b/doc/source/whatsnew/v0.13.1.rst
index 8c85868e1aedb..483dd15a8467a 100644
--- a/doc/source/whatsnew/v0.13.1.rst
+++ b/doc/source/whatsnew/v0.13.1.rst
@@ -24,8 +24,8 @@ Highlights include:
.. warning::
0.13.1 fixes a bug that was caused by a combination of having numpy < 1.8, and doing
- chained assignment on a string-like array. Please review :ref:`the docs`,
- chained indexing can have unexpected results and should generally be avoided.
+ chained assignment on a string-like array.
+ Chained indexing can have unexpected results and should generally be avoided.
This would previously segfault:
diff --git a/doc/source/whatsnew/v0.15.1.rst b/doc/source/whatsnew/v0.15.1.rst
index 09b59f35972cd..765201996d544 100644
--- a/doc/source/whatsnew/v0.15.1.rst
+++ b/doc/source/whatsnew/v0.15.1.rst
@@ -92,7 +92,7 @@ API changes
.. code-block:: ipython
- In [4]: gr.apply(sum)
+ In [4]: gr.apply("sum")
Out[4]:
joe
jim
@@ -102,9 +102,8 @@ API changes
current behavior:
.. ipython:: python
- :okwarning:
- gr.apply(sum)
+ gr.apply("sum")
- Support for slicing with monotonic decreasing indexes, even if ``start`` or ``stop`` is
not found in the index (:issue:`7860`):
diff --git a/doc/source/whatsnew/v0.20.0.rst b/doc/source/whatsnew/v0.20.0.rst
index 09bf5428d0432..f63db945165e7 100644
--- a/doc/source/whatsnew/v0.20.0.rst
+++ b/doc/source/whatsnew/v0.20.0.rst
@@ -369,7 +369,6 @@ Experimental support has been added to export ``DataFrame.style`` formats to Exc
For example, after running the following, ``styled.xlsx`` renders as below:
.. ipython:: python
- :okwarning:
np.random.seed(24)
df = pd.DataFrame({'A': np.linspace(1, 10, 10)})
@@ -379,7 +378,7 @@ For example, after running the following, ``styled.xlsx`` renders as below:
df.iloc[0, 2] = np.nan
df
styled = (df.style
- .applymap(lambda val: 'color:red;' if val < 0 else 'color:black;')
+ .map(lambda val: 'color:red;' if val < 0 else 'color:black;')
.highlight_max())
styled.to_excel('styled.xlsx', engine='openpyxl')
diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst
index cb12962256a55..c63d047f03823 100644
--- a/doc/source/whatsnew/v0.24.0.rst
+++ b/doc/source/whatsnew/v0.24.0.rst
@@ -840,7 +840,7 @@ then all the columns are dummy-encoded, and a :class:`SparseDataFrame` was retur
In [2]: df = pd.DataFrame({"A": [1, 2], "B": ['a', 'b'], "C": ['a', 'a']})
In [3]: type(pd.get_dummies(df, sparse=True))
- Out[3]: pandas.core.frame.DataFrame
+ Out[3]: pandas.DataFrame
In [4]: type(pd.get_dummies(df[['B', 'C']], sparse=True))
Out[4]: pandas.core.sparse.frame.SparseDataFrame
diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst
index 94a8ee7cd1a5d..5dbf6f1c60598 100755
--- a/doc/source/whatsnew/v1.0.0.rst
+++ b/doc/source/whatsnew/v1.0.0.rst
@@ -414,7 +414,7 @@ Extended verbose info output for :class:`~pandas.DataFrame`
... "text_col": ["a", "b", "c"],
... "float_col": [0.0, 0.1, 0.2]})
In [2]: df.info(verbose=True)
-
+
RangeIndex: 3 entries, 0 to 2
Data columns (total 3 columns):
int_col 3 non-null int64
diff --git a/doc/source/whatsnew/v1.3.1.rst b/doc/source/whatsnew/v1.3.1.rst
index a57995eb0db9a..f3b21554e668c 100644
--- a/doc/source/whatsnew/v1.3.1.rst
+++ b/doc/source/whatsnew/v1.3.1.rst
@@ -14,7 +14,7 @@ including other versions of pandas.
Fixed regressions
~~~~~~~~~~~~~~~~~
-- Pandas could not be built on PyPy (:issue:`42355`)
+- pandas could not be built on PyPy (:issue:`42355`)
- :class:`DataFrame` constructed with an older version of pandas could not be unpickled (:issue:`42345`)
- Performance regression in constructing a :class:`DataFrame` from a dictionary of dictionaries (:issue:`42248`)
- Fixed regression in :meth:`DataFrame.agg` dropping values when the DataFrame had an Extension Array dtype, a duplicate index, and ``axis=1`` (:issue:`42380`)
diff --git a/doc/source/whatsnew/v1.4.0.rst b/doc/source/whatsnew/v1.4.0.rst
index ec0b04f8b3b77..91953f693190c 100644
--- a/doc/source/whatsnew/v1.4.0.rst
+++ b/doc/source/whatsnew/v1.4.0.rst
@@ -607,7 +607,7 @@ Deprecated Int64Index, UInt64Index & Float64Index
:class:`Int64Index`, :class:`UInt64Index` and :class:`Float64Index` have been
deprecated in favor of the base :class:`Index` class and will be removed in
-Pandas 2.0 (:issue:`43028`).
+pandas 2.0 (:issue:`43028`).
For constructing a numeric index, you can use the base :class:`Index` class
instead specifying the data type (which will also work on older pandas
diff --git a/doc/source/whatsnew/v1.5.0.rst b/doc/source/whatsnew/v1.5.0.rst
index 8fa1361cc30c1..43aa63c284f38 100644
--- a/doc/source/whatsnew/v1.5.0.rst
+++ b/doc/source/whatsnew/v1.5.0.rst
@@ -63,7 +63,7 @@ We recommend installing the latest version of PyArrow to access the most recentl
DataFrame interchange protocol implementation
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Pandas now implement the DataFrame interchange API spec.
+pandas now implement the DataFrame interchange API spec.
See the full details on the API at https://data-apis.org/dataframe-protocol/latest/index.html
The protocol consists of two parts:
@@ -383,7 +383,7 @@ Other enhancements
- Added ``validate`` argument to :meth:`DataFrame.join` (:issue:`46622`)
- Added ``numeric_only`` argument to :meth:`.Resampler.sum`, :meth:`.Resampler.prod`, :meth:`.Resampler.min`, :meth:`.Resampler.max`, :meth:`.Resampler.first`, and :meth:`.Resampler.last` (:issue:`46442`)
- ``times`` argument in :class:`.ExponentialMovingWindow` now accepts ``np.timedelta64`` (:issue:`47003`)
-- :class:`.DataError`, :class:`.SpecificationError`, :class:`.SettingWithCopyError`, :class:`.SettingWithCopyWarning`, :class:`.NumExprClobberingError`, :class:`.UndefinedVariableError`, :class:`.IndexingError`, :class:`.PyperclipException`, :class:`.PyperclipWindowsException`, :class:`.CSSWarning`, :class:`.PossibleDataLossError`, :class:`.ClosedFileError`, :class:`.IncompatibilityWarning`, :class:`.AttributeConflictWarning`, :class:`.DatabaseError`, :class:`.PossiblePrecisionLoss`, :class:`.ValueLabelTypeMismatch`, :class:`.InvalidColumnName`, and :class:`.CategoricalConversionWarning` are now exposed in ``pandas.errors`` (:issue:`27656`)
+- :class:`.DataError`, :class:`.SpecificationError`, ``SettingWithCopyError``, ``SettingWithCopyWarning``, :class:`.NumExprClobberingError`, :class:`.UndefinedVariableError`, :class:`.IndexingError`, :class:`.PyperclipException`, :class:`.PyperclipWindowsException`, :class:`.CSSWarning`, :class:`.PossibleDataLossError`, :class:`.ClosedFileError`, :class:`.IncompatibilityWarning`, :class:`.AttributeConflictWarning`, :class:`.DatabaseError`, :class:`.PossiblePrecisionLoss`, :class:`.ValueLabelTypeMismatch`, :class:`.InvalidColumnName`, and :class:`.CategoricalConversionWarning` are now exposed in ``pandas.errors`` (:issue:`27656`)
- Added ``check_like`` argument to :func:`testing.assert_series_equal` (:issue:`47247`)
- Add support for :meth:`.DataFrameGroupBy.ohlc` and :meth:`.SeriesGroupBy.ohlc` for extension array dtypes (:issue:`37493`)
- Allow reading compressed SAS files with :func:`read_sas` (e.g., ``.sas7bdat.gz`` files)
diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst
index be63da09846c8..cacbf8452ba32 100644
--- a/doc/source/whatsnew/v2.0.0.rst
+++ b/doc/source/whatsnew/v2.0.0.rst
@@ -57,7 +57,7 @@ can it now take all numpy numeric dtypes, i.e.
pd.Index([1, 2, 3], dtype=np.uint16)
pd.Index([1, 2, 3], dtype=np.float32)
-The ability for :class:`Index` to hold the numpy numeric dtypes has meant some changes in Pandas
+The ability for :class:`Index` to hold the numpy numeric dtypes has meant some changes in pandas
functionality. In particular, operations that previously were forced to create 64-bit indexes,
can now create indexes with lower bit sizes, e.g. 32-bit indexes.
@@ -732,7 +732,7 @@ or, if your formats are all ISO8601 (but possibly not identically-formatted) ::
Other API changes
^^^^^^^^^^^^^^^^^
-- The ``freq``, ``tz``, ``nanosecond``, and ``unit`` keywords in the :class:`Timestamp` constructor are now keyword-only (:issue:`45307`, :issue:`32526`)
+- The ``tz``, ``nanosecond``, and ``unit`` keywords in the :class:`Timestamp` constructor are now keyword-only (:issue:`45307`, :issue:`32526`)
- Passing ``nanoseconds`` greater than 999 or less than 0 in :class:`Timestamp` now raises a ``ValueError`` (:issue:`48538`, :issue:`48255`)
- :func:`read_csv`: specifying an incorrect number of columns with ``index_col`` of now raises ``ParserError`` instead of ``IndexError`` when using the c parser.
- Default value of ``dtype`` in :func:`get_dummies` is changed to ``bool`` from ``uint8`` (:issue:`45848`)
diff --git a/doc/source/whatsnew/v2.1.0.rst b/doc/source/whatsnew/v2.1.0.rst
index 51b4c4f297b07..495c8244142f9 100644
--- a/doc/source/whatsnew/v2.1.0.rst
+++ b/doc/source/whatsnew/v2.1.0.rst
@@ -67,7 +67,7 @@ DataFrame reductions preserve extension dtypes
In previous versions of pandas, the results of DataFrame reductions
(:meth:`DataFrame.sum` :meth:`DataFrame.mean` etc.) had NumPy dtypes, even when the DataFrames
-were of extension dtypes. Pandas can now keep the dtypes when doing reductions over DataFrame
+were of extension dtypes. pandas can now keep the dtypes when doing reductions over DataFrame
columns with a common dtype (:issue:`52788`).
*Old Behavior*
@@ -432,7 +432,7 @@ In a future version, these will raise an error and you should cast to a common d
In [3]: ser[0] = 'not an int64'
FutureWarning:
- Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas.
+ Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas.
Value 'not an int64' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.
In [4]: ser
diff --git a/doc/source/whatsnew/v2.1.4.rst b/doc/source/whatsnew/v2.1.4.rst
index 57b83a294963b..73b1103c1bd37 100644
--- a/doc/source/whatsnew/v2.1.4.rst
+++ b/doc/source/whatsnew/v2.1.4.rst
@@ -42,4 +42,4 @@ Bug fixes
Contributors
~~~~~~~~~~~~
-.. contributors:: v2.1.3..v2.1.4|HEAD
+.. contributors:: v2.1.3..v2.1.4
diff --git a/doc/source/whatsnew/v2.2.0.rst b/doc/source/whatsnew/v2.2.0.rst
index 649ad37a56b35..329ef2859f56f 100644
--- a/doc/source/whatsnew/v2.2.0.rst
+++ b/doc/source/whatsnew/v2.2.0.rst
@@ -1,7 +1,7 @@
.. _whatsnew_220:
-What's new in 2.2.0 (Month XX, 2024)
-------------------------------------
+What's new in 2.2.0 (January 19, 2024)
+--------------------------------------
These are the changes in pandas 2.2.0. See :ref:`release` for a full changelog
including other versions of pandas.
@@ -188,6 +188,26 @@ For a full list of ADBC drivers and their development status, see the `ADBC Driv
Implementation Status `_
documentation.
+.. _whatsnew_220.enhancements.case_when:
+
+Create a pandas Series based on one or more conditions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The :meth:`Series.case_when` function has been added to create a Series object based on one or more conditions. (:issue:`39154`)
+
+.. ipython:: python
+
+ import pandas as pd
+
+ df = pd.DataFrame(dict(a=[1, 2, 3], b=[4, 5, 6]))
+ default=pd.Series('default', index=df.index)
+ default.case_when(
+ caselist=[
+ (df.a == 1, 'first'), # condition, replacement
+ (df.a.gt(1) & df.b.eq(5), 'second'), # condition, replacement
+ ],
+ )
+
.. _whatsnew_220.enhancements.to_numpy_ea:
``to_numpy`` for NumPy nullable and Arrow types converts to suitable NumPy dtype
@@ -251,6 +271,14 @@ DataFrame. (:issue:`54938`)
)
series.struct.explode()
+Use :meth:`Series.struct.field` to index into a (possible nested)
+struct field.
+
+
+.. ipython:: python
+
+ series.struct.field("project")
+
.. _whatsnew_220.enhancements.list_accessor:
Series.list accessor for PyArrow list data
@@ -313,8 +341,9 @@ Other enhancements
- :meth:`DataFrame.apply` now allows the usage of numba (via ``engine="numba"``) to JIT compile the passed function, allowing for potential speedups (:issue:`54666`)
- :meth:`ExtensionArray._explode` interface method added to allow extension type implementations of the ``explode`` method (:issue:`54833`)
- :meth:`ExtensionArray.duplicated` added to allow extension type implementations of the ``duplicated`` method (:issue:`55255`)
-- :meth:`Series.ffill`, :meth:`Series.bfill`, :meth:`DataFrame.ffill`, and :meth:`DataFrame.bfill` have gained the argument ``limit_area`` (:issue:`56492`)
+- :meth:`Series.ffill`, :meth:`Series.bfill`, :meth:`DataFrame.ffill`, and :meth:`DataFrame.bfill` have gained the argument ``limit_area``; 3rd party :class:`.ExtensionArray` authors need to add this argument to the method ``_pad_or_backfill`` (:issue:`56492`)
- Allow passing ``read_only``, ``data_only`` and ``keep_links`` arguments to openpyxl using ``engine_kwargs`` of :func:`read_excel` (:issue:`55027`)
+- Implement :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` for :class:`ArrowDtype` and masked dtypes (:issue:`56267`)
- Implement masked algorithms for :meth:`Series.value_counts` (:issue:`54984`)
- Implemented :meth:`Series.dt` methods and attributes for :class:`ArrowDtype` with ``pyarrow.duration`` type (:issue:`52284`)
- Implemented :meth:`Series.str.extract` for :class:`ArrowDtype` (:issue:`56268`)
@@ -407,12 +436,6 @@ index levels when joining on two indexes with different levels (:issue:`34133`).
result
-.. ---------------------------------------------------------------------------
-.. _whatsnew_220.api_breaking:
-
-Backwards incompatible API changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
.. _whatsnew_220.api_breaking.deps:
Increased minimum versions for dependencies
@@ -596,6 +619,8 @@ For example:
pd.date_range('2020-01-01', periods=3, freq='QE-NOV')
+.. _whatsnew_220.silent_downcasting:
+
Deprecated automatic downcasting
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -633,14 +658,15 @@ Other Deprecations
- Changed :meth:`Timedelta.resolution_string` to return ``h``, ``min``, ``s``, ``ms``, ``us``, and ``ns`` instead of ``H``, ``T``, ``S``, ``L``, ``U``, and ``N``, for compatibility with respective deprecations in frequency aliases (:issue:`52536`)
- Deprecated :attr:`offsets.Day.delta`, :attr:`offsets.Hour.delta`, :attr:`offsets.Minute.delta`, :attr:`offsets.Second.delta`, :attr:`offsets.Milli.delta`, :attr:`offsets.Micro.delta`, :attr:`offsets.Nano.delta`, use ``pd.Timedelta(obj)`` instead (:issue:`55498`)
- Deprecated :func:`pandas.api.types.is_interval` and :func:`pandas.api.types.is_period`, use ``isinstance(obj, pd.Interval)`` and ``isinstance(obj, pd.Period)`` instead (:issue:`55264`)
-- Deprecated :func:`pd.core.internals.api.make_block`, use public APIs instead (:issue:`40226`)
- Deprecated :func:`read_gbq` and :meth:`DataFrame.to_gbq`. Use ``pandas_gbq.read_gbq`` and ``pandas_gbq.to_gbq`` instead https://pandas-gbq.readthedocs.io/en/latest/api.html (:issue:`55525`)
- Deprecated :meth:`.DataFrameGroupBy.fillna` and :meth:`.SeriesGroupBy.fillna`; use :meth:`.DataFrameGroupBy.ffill`, :meth:`.DataFrameGroupBy.bfill` for forward and backward filling or :meth:`.DataFrame.fillna` to fill with a single value (or the Series equivalents) (:issue:`55718`)
+- Deprecated :meth:`DateOffset.is_anchored`, use ``obj.n == 1`` for non-Tick subclasses (for Tick this was always False) (:issue:`55388`)
- Deprecated :meth:`DatetimeArray.__init__` and :meth:`TimedeltaArray.__init__`, use :func:`array` instead (:issue:`55623`)
- Deprecated :meth:`Index.format`, use ``index.astype(str)`` or ``index.map(formatter)`` instead (:issue:`55413`)
- Deprecated :meth:`Series.ravel`, the underlying array is already 1D, so ravel is not necessary (:issue:`52511`)
- Deprecated :meth:`Series.resample` and :meth:`DataFrame.resample` with a :class:`PeriodIndex` (and the 'convention' keyword), convert to :class:`DatetimeIndex` (with ``.to_timestamp()``) before resampling instead (:issue:`53481`)
- Deprecated :meth:`Series.view`, use :meth:`Series.astype` instead to change the dtype (:issue:`20251`)
+- Deprecated :meth:`offsets.Tick.is_anchored`, use ``False`` instead (:issue:`55388`)
- Deprecated ``core.internals`` members ``Block``, ``ExtensionBlock``, and ``DatetimeTZBlock``, use public APIs instead (:issue:`55139`)
- Deprecated ``year``, ``month``, ``quarter``, ``day``, ``hour``, ``minute``, and ``second`` keywords in the :class:`PeriodIndex` constructor, use :meth:`PeriodIndex.from_fields` instead (:issue:`55960`)
- Deprecated accepting a type as an argument in :meth:`Index.view`, call without any arguments instead (:issue:`55709`)
@@ -691,6 +717,7 @@ Other Deprecations
- Deprecated the extension test classes ``BaseNoReduceTests``, ``BaseBooleanReduceTests``, and ``BaseNumericReduceTests``, use ``BaseReduceTests`` instead (:issue:`54663`)
- Deprecated the option ``mode.data_manager`` and the ``ArrayManager``; only the ``BlockManager`` will be available in future versions (:issue:`55043`)
- Deprecated the previous implementation of :class:`DataFrame.stack`; specify ``future_stack=True`` to adopt the future version (:issue:`53515`)
+-
.. ---------------------------------------------------------------------------
.. _whatsnew_220.performance:
@@ -732,6 +759,7 @@ Categorical
^^^^^^^^^^^
- :meth:`Categorical.isin` raising ``InvalidIndexError`` for categorical containing overlapping :class:`Interval` values (:issue:`34974`)
- Bug in :meth:`CategoricalDtype.__eq__` returning ``False`` for unordered categorical data with mixed types (:issue:`55468`)
+- Bug when casting ``pa.dictionary`` to :class:`CategoricalDtype` using a ``pa.DictionaryArray`` as categories (:issue:`56672`)
Datetimelike
^^^^^^^^^^^^
@@ -777,14 +805,18 @@ Timezones
Numeric
^^^^^^^
- Bug in :func:`read_csv` with ``engine="pyarrow"`` causing rounding errors for large integers (:issue:`52505`)
+- Bug in :meth:`Series.__floordiv__` and :meth:`Series.__truediv__` for :class:`ArrowDtype` with integral dtypes raising for large divisors (:issue:`56706`)
- Bug in :meth:`Series.__floordiv__` for :class:`ArrowDtype` with integral dtypes raising for large values (:issue:`56645`)
- Bug in :meth:`Series.pow` not filling missing values correctly (:issue:`55512`)
+- Bug in :meth:`Series.replace` and :meth:`DataFrame.replace` matching float ``0.0`` with ``False`` and vice versa (:issue:`55398`)
+- Bug in :meth:`Series.round` raising for nullable boolean dtype (:issue:`55936`)
Conversion
^^^^^^^^^^
- Bug in :meth:`DataFrame.astype` when called with ``str`` on unpickled array - the array might change in-place (:issue:`54654`)
- Bug in :meth:`DataFrame.astype` where ``errors="ignore"`` had no effect for extension types (:issue:`54654`)
- Bug in :meth:`Series.convert_dtypes` not converting all NA column to ``null[pyarrow]`` (:issue:`55346`)
+- Bug in :meth:``DataFrame.loc`` was not throwing "incompatible dtype warning" (see `PDEP6 `_) when assigning a ``Series`` with a different dtype using a full column setter (e.g. ``df.loc[:, 'a'] = incompatible_value``) (:issue:`39584`)
Strings
^^^^^^^
@@ -794,6 +826,7 @@ Strings
- Bug in :meth:`Index.str.cat` always casting result to object dtype (:issue:`56157`)
- Bug in :meth:`Series.__mul__` for :class:`ArrowDtype` with ``pyarrow.string`` dtype and ``string[pyarrow]`` for the pyarrow backend (:issue:`51970`)
- Bug in :meth:`Series.str.find` when ``start < 0`` for :class:`ArrowDtype` with ``pyarrow.string`` (:issue:`56411`)
+- Bug in :meth:`Series.str.fullmatch` when ``dtype=pandas.ArrowDtype(pyarrow.string()))`` allows partial matches when regex ends in literal //$ (:issue:`56652`)
- Bug in :meth:`Series.str.replace` when ``n < 0`` for :class:`ArrowDtype` with ``pyarrow.string`` (:issue:`56404`)
- Bug in :meth:`Series.str.startswith` and :meth:`Series.str.endswith` with arguments of type ``tuple[str, ...]`` for :class:`ArrowDtype` with ``pyarrow.string`` dtype (:issue:`56579`)
- Bug in :meth:`Series.str.startswith` and :meth:`Series.str.endswith` with arguments of type ``tuple[str, ...]`` for ``string[pyarrow]`` (:issue:`54942`)
@@ -804,6 +837,7 @@ Interval
- Bug in :class:`Interval` ``__repr__`` not displaying UTC offsets for :class:`Timestamp` bounds. Additionally the hour, minute and second components will now be shown (:issue:`55015`)
- Bug in :meth:`IntervalIndex.factorize` and :meth:`Series.factorize` with :class:`IntervalDtype` with datetime64 or timedelta64 intervals not preserving non-nanosecond units (:issue:`56099`)
- Bug in :meth:`IntervalIndex.from_arrays` when passed ``datetime64`` or ``timedelta64`` arrays with mismatched resolutions constructing an invalid ``IntervalArray`` object (:issue:`55714`)
+- Bug in :meth:`IntervalIndex.from_tuples` raising if subtype is a nullable extension dtype (:issue:`56765`)
- Bug in :meth:`IntervalIndex.get_indexer` with datetime or timedelta intervals incorrectly matching on integer targets (:issue:`47772`)
- Bug in :meth:`IntervalIndex.get_indexer` with timezone-aware datetime intervals incorrectly matching on a sequence of timezone-naive targets (:issue:`47772`)
- Bug in setting values on a :class:`Series` with an :class:`IntervalIndex` using a slice incorrectly raising (:issue:`54722`)
@@ -835,6 +869,7 @@ I/O
- Bug in :func:`read_json` not handling dtype conversion properly if ``infer_string`` is set (:issue:`56195`)
- Bug in :meth:`DataFrame.to_excel`, with ``OdsWriter`` (``ods`` files) writing Boolean/string value (:issue:`54994`)
- Bug in :meth:`DataFrame.to_hdf` and :func:`read_hdf` with ``datetime64`` dtypes with non-nanosecond resolution failing to round-trip correctly (:issue:`55622`)
+- Bug in :meth:`DataFrame.to_stata` raising for extension dtypes (:issue:`54671`)
- Bug in :meth:`~pandas.read_excel` with ``engine="odf"`` (``ods`` files) when a string cell contains an annotation (:issue:`55200`)
- Bug in :meth:`~pandas.read_excel` with an ODS file without cached formatted cell for float values (:issue:`55219`)
- Bug where :meth:`DataFrame.to_json` would raise an ``OverflowError`` instead of a ``TypeError`` with unsupported NumPy types (:issue:`55403`)
@@ -853,7 +888,6 @@ Plotting
Groupby/resample/rolling
^^^^^^^^^^^^^^^^^^^^^^^^
-- Bug in :class:`.Rolling` where duplicate datetimelike indexes are treated as consecutive rather than equal with ``closed='left'`` and ``closed='neither'`` (:issue:`20712`)
- Bug in :meth:`.DataFrameGroupBy.idxmin`, :meth:`.DataFrameGroupBy.idxmax`, :meth:`.SeriesGroupBy.idxmin`, and :meth:`.SeriesGroupBy.idxmax` would not retain :class:`.Categorical` dtype when the index was a :class:`.CategoricalIndex` that contained NA values (:issue:`54234`)
- Bug in :meth:`.DataFrameGroupBy.transform` and :meth:`.SeriesGroupBy.transform` when ``observed=False`` and ``f="idxmin"`` or ``f="idxmax"`` would incorrectly raise on unobserved categories (:issue:`54234`)
- Bug in :meth:`.DataFrameGroupBy.value_counts` and :meth:`.SeriesGroupBy.value_counts` could result in incorrect sorting if the columns of the DataFrame or name of the Series are integers (:issue:`55951`)
@@ -862,10 +896,12 @@ Groupby/resample/rolling
- Bug in :meth:`DataFrame.asfreq` and :meth:`Series.asfreq` with a :class:`DatetimeIndex` with non-nanosecond resolution incorrectly converting to nanosecond resolution (:issue:`55958`)
- Bug in :meth:`DataFrame.ewm` when passed ``times`` with non-nanosecond ``datetime64`` or :class:`DatetimeTZDtype` dtype (:issue:`56262`)
- Bug in :meth:`DataFrame.groupby` and :meth:`Series.groupby` where grouping by a combination of ``Decimal`` and NA values would fail when ``sort=True`` (:issue:`54847`)
+- Bug in :meth:`DataFrame.groupby` for DataFrame subclasses when selecting a subset of columns to apply the function to (:issue:`56761`)
- Bug in :meth:`DataFrame.resample` not respecting ``closed`` and ``label`` arguments for :class:`~pandas.tseries.offsets.BusinessDay` (:issue:`55282`)
- Bug in :meth:`DataFrame.resample` when resampling on a :class:`ArrowDtype` of ``pyarrow.timestamp`` or ``pyarrow.duration`` type (:issue:`55989`)
- Bug in :meth:`DataFrame.resample` where bin edges were not correct for :class:`~pandas.tseries.offsets.BusinessDay` (:issue:`55281`)
- Bug in :meth:`DataFrame.resample` where bin edges were not correct for :class:`~pandas.tseries.offsets.MonthBegin` (:issue:`55271`)
+- Bug in :meth:`DataFrame.rolling` and :meth:`Series.rolling` where duplicate datetimelike indexes are treated as consecutive rather than equal with ``closed='left'`` and ``closed='neither'`` (:issue:`20712`)
- Bug in :meth:`DataFrame.rolling` and :meth:`Series.rolling` where either the ``index`` or ``on`` column was :class:`ArrowDtype` with ``pyarrow.timestamp`` type (:issue:`55849`)
Reshaping
@@ -877,6 +913,7 @@ Reshaping
- Bug in :func:`merge_asof` when using a :class:`Timedelta` tolerance on a :class:`ArrowDtype` column (:issue:`56486`)
- Bug in :func:`merge` not raising when merging datetime columns with timedelta columns (:issue:`56455`)
- Bug in :func:`merge` not raising when merging string columns with numeric columns (:issue:`56441`)
+- Bug in :func:`merge` not sorting for new string dtype (:issue:`56442`)
- Bug in :func:`merge` returning columns in incorrect order when left and/or right is empty (:issue:`51929`)
- Bug in :meth:`DataFrame.melt` where an exception was raised if ``var_name`` was not a string (:issue:`55948`)
- Bug in :meth:`DataFrame.melt` where it would not preserve the datetime (:issue:`55254`)
@@ -886,11 +923,13 @@ Reshaping
Sparse
^^^^^^
-- Bug in :meth:`SparseArray.take` when using a different fill value than the array's fill value (:issue:`55181`)
+- Bug in :meth:`arrays.SparseArray.take` when using a different fill value than the array's fill value (:issue:`55181`)
Other
^^^^^
+- :meth:`DataFrame.__dataframe__` did not support pyarrow large strings (:issue:`56702`)
- Bug in :func:`DataFrame.describe` when formatting percentiles in the resulting percentile 99.999% is rounded to 100% (:issue:`55765`)
+- Bug in :func:`api.interchange.from_dataframe` where it raised ``NotImplementedError`` when handling empty string columns (:issue:`56703`)
- Bug in :func:`cut` and :func:`qcut` with ``datetime64`` dtype values with non-nanosecond units incorrectly returning nanosecond-unit bins (:issue:`56101`)
- Bug in :func:`cut` incorrectly allowing cutting of timezone-aware datetimes with timezone-naive bins (:issue:`54964`)
- Bug in :func:`infer_freq` and :meth:`DatetimeIndex.inferred_freq` with weekly frequencies and non-nanosecond resolutions (:issue:`55609`)
@@ -901,10 +940,12 @@ Other
- Bug in rendering a :class:`Series` with a :class:`MultiIndex` when one of the index level's names is 0 not having that name displayed (:issue:`55415`)
- Bug in the error message when assigning an empty :class:`DataFrame` to a column (:issue:`55956`)
- Bug when time-like strings were being cast to :class:`ArrowDtype` with ``pyarrow.time64`` type (:issue:`56463`)
-
+- Fixed a spurious deprecation warning from ``numba`` >= 0.58.0 when passing a numpy ufunc in :class:`core.window.Rolling.apply` with ``engine="numba"`` (:issue:`55247`)
.. ---------------------------------------------------------------------------
.. _whatsnew_220.contributors:
Contributors
~~~~~~~~~~~~
+
+.. contributors:: v2.1.4..v2.2.0
diff --git a/doc/source/whatsnew/v2.2.1.rst b/doc/source/whatsnew/v2.2.1.rst
new file mode 100644
index 0000000000000..310dd921e44f6
--- /dev/null
+++ b/doc/source/whatsnew/v2.2.1.rst
@@ -0,0 +1,90 @@
+.. _whatsnew_221:
+
+What's new in 2.2.1 (February 22, 2024)
+---------------------------------------
+
+These are the changes in pandas 2.2.1. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_221.enhancements:
+
+Enhancements
+~~~~~~~~~~~~
+- Added ``pyarrow`` pip extra so users can install pandas and pyarrow with pip with ``pip install pandas[pyarrow]`` (:issue:`54466`)
+
+.. _whatsnew_221.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+- Fixed memory leak in :func:`read_csv` (:issue:`57039`)
+- Fixed performance regression in :meth:`Series.combine_first` (:issue:`55845`)
+- Fixed regression causing overflow for near-minimum timestamps (:issue:`57150`)
+- Fixed regression in :func:`concat` changing long-standing behavior that always sorted the non-concatenation axis when the axis was a :class:`DatetimeIndex` (:issue:`57006`)
+- Fixed regression in :func:`merge_ordered` raising ``TypeError`` for ``fill_method="ffill"`` and ``how="left"`` (:issue:`57010`)
+- Fixed regression in :func:`pandas.testing.assert_series_equal` defaulting to ``check_exact=True`` when checking the :class:`Index` (:issue:`57067`)
+- Fixed regression in :func:`read_json` where an :class:`Index` would be returned instead of a :class:`RangeIndex` (:issue:`57429`)
+- Fixed regression in :func:`wide_to_long` raising an ``AttributeError`` for string columns (:issue:`57066`)
+- Fixed regression in :meth:`.DataFrameGroupBy.idxmin`, :meth:`.DataFrameGroupBy.idxmax`, :meth:`.SeriesGroupBy.idxmin`, :meth:`.SeriesGroupBy.idxmax` ignoring the ``skipna`` argument (:issue:`57040`)
+- Fixed regression in :meth:`.DataFrameGroupBy.idxmin`, :meth:`.DataFrameGroupBy.idxmax`, :meth:`.SeriesGroupBy.idxmin`, :meth:`.SeriesGroupBy.idxmax` where values containing the minimum or maximum value for the dtype could produce incorrect results (:issue:`57040`)
+- Fixed regression in :meth:`CategoricalIndex.difference` raising ``KeyError`` when other contains null values other than NaN (:issue:`57318`)
+- Fixed regression in :meth:`DataFrame.groupby` raising ``ValueError`` when grouping by a :class:`Series` in some cases (:issue:`57276`)
+- Fixed regression in :meth:`DataFrame.loc` raising ``IndexError`` for non-unique, masked dtype indexes where result has more than 10,000 rows (:issue:`57027`)
+- Fixed regression in :meth:`DataFrame.loc` which was unnecessarily throwing "incompatible dtype warning" when expanding with partial row indexer and multiple columns (see `PDEP6 `_) (:issue:`56503`)
+- Fixed regression in :meth:`DataFrame.map` with ``na_action="ignore"`` not being respected for NumPy nullable and :class:`ArrowDtypes` (:issue:`57316`)
+- Fixed regression in :meth:`DataFrame.merge` raising ``ValueError`` for certain types of 3rd-party extension arrays (:issue:`57316`)
+- Fixed regression in :meth:`DataFrame.query` with all ``NaT`` column with object dtype (:issue:`57068`)
+- Fixed regression in :meth:`DataFrame.shift` raising ``AssertionError`` for ``axis=1`` and empty :class:`DataFrame` (:issue:`57301`)
+- Fixed regression in :meth:`DataFrame.sort_index` not producing a stable sort for a index with duplicates (:issue:`57151`)
+- Fixed regression in :meth:`DataFrame.to_dict` with ``orient='list'`` and datetime or timedelta types returning integers (:issue:`54824`)
+- Fixed regression in :meth:`DataFrame.to_json` converting nullable integers to floats (:issue:`57224`)
+- Fixed regression in :meth:`DataFrame.to_sql` when ``method="multi"`` is passed and the dialect type is not Oracle (:issue:`57310`)
+- Fixed regression in :meth:`DataFrame.transpose` with nullable extension dtypes not having F-contiguous data potentially causing exceptions when used (:issue:`57315`)
+- Fixed regression in :meth:`DataFrame.update` emitting incorrect warnings about downcasting (:issue:`57124`)
+- Fixed regression in :meth:`DataFrameGroupBy.idxmin`, :meth:`DataFrameGroupBy.idxmax`, :meth:`SeriesGroupBy.idxmin`, :meth:`SeriesGroupBy.idxmax` ignoring the ``skipna`` argument (:issue:`57040`)
+- Fixed regression in :meth:`DataFrameGroupBy.idxmin`, :meth:`DataFrameGroupBy.idxmax`, :meth:`SeriesGroupBy.idxmin`, :meth:`SeriesGroupBy.idxmax` where values containing the minimum or maximum value for the dtype could produce incorrect results (:issue:`57040`)
+- Fixed regression in :meth:`ExtensionArray.to_numpy` raising for non-numeric masked dtypes (:issue:`56991`)
+- Fixed regression in :meth:`Index.join` raising ``TypeError`` when joining an empty index to a non-empty index containing mixed dtype values (:issue:`57048`)
+- Fixed regression in :meth:`Series.astype` introducing decimals when converting from integer with missing values to string dtype (:issue:`57418`)
+- Fixed regression in :meth:`Series.pct_change` raising a ``ValueError`` for an empty :class:`Series` (:issue:`57056`)
+- Fixed regression in :meth:`Series.to_numpy` when dtype is given as float and the data contains NaNs (:issue:`57121`)
+- Fixed regression in addition or subtraction of :class:`DateOffset` objects with millisecond components to ``datetime64`` :class:`Index`, :class:`Series`, or :class:`DataFrame` (:issue:`57529`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_221.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+- Fixed bug in :func:`pandas.api.interchange.from_dataframe` which was raising for Nullable integers (:issue:`55069`)
+- Fixed bug in :func:`pandas.api.interchange.from_dataframe` which was raising for empty inputs (:issue:`56700`)
+- Fixed bug in :func:`pandas.api.interchange.from_dataframe` which wasn't converting columns names to strings (:issue:`55069`)
+- Fixed bug in :meth:`DataFrame.__getitem__` for empty :class:`DataFrame` with Copy-on-Write enabled (:issue:`57130`)
+- Fixed bug in :meth:`PeriodIndex.asfreq` which was silently converting frequencies which are not supported as period frequencies instead of raising an error (:issue:`56945`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_221.other:
+
+Other
+~~~~~
+
+.. note::
+
+ The ``DeprecationWarning`` that was raised when pandas was imported without PyArrow being
+ installed has been removed. This decision was made because the warning was too noisy for too
+ many users and a lot of feedback was collected about the decision to make PyArrow a required
+ dependency. Pandas is currently considering the decision whether or not PyArrow should be added
+ as a hard dependency in 3.0. Interested users can follow the discussion
+ `here `_.
+
+- Added the argument ``skipna`` to :meth:`DataFrameGroupBy.first`, :meth:`DataFrameGroupBy.last`, :meth:`SeriesGroupBy.first`, and :meth:`SeriesGroupBy.last`; achieving ``skipna=False`` used to be available via :meth:`DataFrameGroupBy.nth`, but the behavior was changed in pandas 2.0.0 (:issue:`57019`)
+- Added the argument ``skipna`` to :meth:`Resampler.first`, :meth:`Resampler.last` (:issue:`57019`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_221.contributors:
+
+Contributors
+~~~~~~~~~~~~
+
+.. contributors:: v2.2.0..v2.2.1|HEAD
diff --git a/doc/source/whatsnew/v2.2.2.rst b/doc/source/whatsnew/v2.2.2.rst
new file mode 100644
index 0000000000000..0dac3660c76b2
--- /dev/null
+++ b/doc/source/whatsnew/v2.2.2.rst
@@ -0,0 +1,42 @@
+.. _whatsnew_222:
+
+What's new in 2.2.2 (April XX, 2024)
+---------------------------------------
+
+These are the changes in pandas 2.2.2. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_222.regressions:
+
+Fixed regressions
+~~~~~~~~~~~~~~~~~
+- :meth:`DataFrame.__dataframe__` was producing incorrect data buffers when the a column's type was a pandas nullable on with missing values (:issue:`56702`)
+- :meth:`DataFrame.__dataframe__` was producing incorrect data buffers when the a column's type was a pyarrow nullable on with missing values (:issue:`57664`)
+- Avoid issuing a spurious ``DeprecationWarning`` when a custom :class:`DataFrame` or :class:`Series` subclass method is called (:issue:`57553`)
+- Fixed regression in precision of :func:`to_datetime` with string and ``unit`` input (:issue:`57051`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_222.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+- :meth:`DataFrame.__dataframe__` was producing incorrect data buffers when the column's type was nullable boolean (:issue:`55332`)
+- :meth:`DataFrame.__dataframe__` was showing bytemask instead of bitmask for ``'string[pyarrow]'`` validity buffer (:issue:`57762`)
+- :meth:`DataFrame.__dataframe__` was showing non-null validity buffer (instead of ``None``) ``'string[pyarrow]'`` without missing values (:issue:`57761`)
+- :meth:`DataFrame.to_sql` was failing to find the right table when using the schema argument (:issue:`57539`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_222.other:
+
+Other
+~~~~~
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_222.contributors:
+
+Contributors
+~~~~~~~~~~~~
diff --git a/doc/source/whatsnew/v2.3.0.rst b/doc/source/whatsnew/v2.3.0.rst
deleted file mode 100644
index 1f1b0c7d7195a..0000000000000
--- a/doc/source/whatsnew/v2.3.0.rst
+++ /dev/null
@@ -1,219 +0,0 @@
-.. _whatsnew_230:
-
-What's new in 2.3.0 (Month XX, 2024)
-------------------------------------
-
-These are the changes in pandas 2.2.0. See :ref:`release` for a full changelog
-including other versions of pandas.
-
-{{ header }}
-
-.. ---------------------------------------------------------------------------
-.. _whatsnew_230.enhancements:
-
-Enhancements
-~~~~~~~~~~~~
-
-.. _whatsnew_230.enhancements.enhancement1:
-
-enhancement1
-^^^^^^^^^^^^
-
-.. _whatsnew_230.enhancements.enhancement2:
-
-enhancement2
-^^^^^^^^^^^^
-
-.. _whatsnew_230.enhancements.other:
-
-Other enhancements
-^^^^^^^^^^^^^^^^^^
--
--
-
-.. ---------------------------------------------------------------------------
-.. _whatsnew_230.notable_bug_fixes:
-
-Notable bug fixes
-~~~~~~~~~~~~~~~~~
-
-These are bug fixes that might have notable behavior changes.
-
-.. _whatsnew_230.notable_bug_fixes.notable_bug_fix1:
-
-notable_bug_fix1
-^^^^^^^^^^^^^^^^
-
-.. _whatsnew_230.notable_bug_fixes.notable_bug_fix2:
-
-notable_bug_fix2
-^^^^^^^^^^^^^^^^
-
-.. ---------------------------------------------------------------------------
-.. _whatsnew_230.api_breaking:
-
-Backwards incompatible API changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. _whatsnew_230.api_breaking.deps:
-
-Increased minimum versions for dependencies
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Some minimum supported versions of dependencies were updated.
-If installed, we now require:
-
-+-----------------+-----------------+----------+---------+
-| Package | Minimum Version | Required | Changed |
-+=================+=================+==========+=========+
-| | | X | X |
-+-----------------+-----------------+----------+---------+
-
-For `optional libraries `_ the general recommendation is to use the latest version.
-The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
-Optional libraries below the lowest tested version may still work, but are not considered supported.
-
-+-----------------+---------------------+
-| Package | New Minimum Version |
-+=================+=====================+
-| | |
-+-----------------+---------------------+
-
-See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.
-
-.. _whatsnew_230.api_breaking.other:
-
-Other API changes
-^^^^^^^^^^^^^^^^^
--
--
-
-.. ---------------------------------------------------------------------------
-.. _whatsnew_230.deprecations:
-
-Deprecations
-~~~~~~~~~~~~
--
--
-
-.. ---------------------------------------------------------------------------
-.. _whatsnew_230.performance:
-
-Performance improvements
-~~~~~~~~~~~~~~~~~~~~~~~~
--
--
-
-.. ---------------------------------------------------------------------------
-.. _whatsnew_230.bug_fixes:
-
-Bug fixes
-~~~~~~~~~
-
-Categorical
-^^^^^^^^^^^
--
--
-
-Datetimelike
-^^^^^^^^^^^^
--
--
-
-Timedelta
-^^^^^^^^^
--
--
-
-Timezones
-^^^^^^^^^
--
--
-
-Numeric
-^^^^^^^
--
--
-
-Conversion
-^^^^^^^^^^
--
--
-
-Strings
-^^^^^^^
--
--
-
-Interval
-^^^^^^^^
--
--
-
-Indexing
-^^^^^^^^
--
--
-
-Missing
-^^^^^^^
--
--
-
-MultiIndex
-^^^^^^^^^^
--
--
-
-I/O
-^^^
--
--
-
-Period
-^^^^^^
--
--
-
-Plotting
-^^^^^^^^
--
--
-
-Groupby/resample/rolling
-^^^^^^^^^^^^^^^^^^^^^^^^
--
--
-
-Reshaping
-^^^^^^^^^
--
--
-
-Sparse
-^^^^^^
--
--
-
-ExtensionArray
-^^^^^^^^^^^^^^
--
--
-
-Styler
-^^^^^^
--
--
-
-Other
-^^^^^
-
-.. ***DO NOT USE THIS SECTION***
-
--
--
-
-.. ---------------------------------------------------------------------------
-.. _whatsnew_230.contributors:
-
-Contributors
-~~~~~~~~~~~~
diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst
new file mode 100644
index 0000000000000..74a19472ec835
--- /dev/null
+++ b/doc/source/whatsnew/v3.0.0.rst
@@ -0,0 +1,459 @@
+.. _whatsnew_230:
+
+What's new in 3.0.0 (Month XX, 2024)
+------------------------------------
+
+These are the changes in pandas 3.0.0. See :ref:`release` for a full changelog
+including other versions of pandas.
+
+{{ header }}
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_300.enhancements:
+
+Enhancements
+~~~~~~~~~~~~
+
+.. _whatsnew_300.enhancements.enhancement1:
+
+enhancement1
+^^^^^^^^^^^^
+
+.. _whatsnew_300.enhancements.enhancement2:
+
+enhancement2
+^^^^^^^^^^^^
+
+.. _whatsnew_300.enhancements.other:
+
+Other enhancements
+^^^^^^^^^^^^^^^^^^
+- :func:`DataFrame.to_excel` now raises an ``UserWarning`` when the character count in a cell exceeds Excel's limitation of 32767 characters (:issue:`56954`)
+- :func:`read_stata` now returns ``datetime64`` resolutions better matching those natively stored in the stata format (:issue:`55642`)
+- :meth:`Styler.set_tooltips` provides alternative method to storing tooltips by using title attribute of td elements. (:issue:`56981`)
+- Allow dictionaries to be passed to :meth:`pandas.Series.str.replace` via ``pat`` parameter (:issue:`51748`)
+- Support passing a :class:`Series` input to :func:`json_normalize` that retains the :class:`Series` :class:`Index` (:issue:`51452`)
+- Users can globally disable any ``PerformanceWarning`` by setting the option ``mode.performance_warnings`` to ``False`` (:issue:`56920`)
+- :meth:`Styler.format_index_names` can now be used to format the index and column names (:issue:`48936` and :issue:`47489`)
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_300.notable_bug_fixes:
+
+Notable bug fixes
+~~~~~~~~~~~~~~~~~
+
+These are bug fixes that might have notable behavior changes.
+
+.. _whatsnew_300.notable_bug_fixes.groupby_unobs_and_na:
+
+Improved behavior in groupby for ``observed=False``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A number of bugs have been fixed due to improved handling of unobserved groups (:issue:`55738`). All remarks in this section equally impact :class:`.SeriesGroupBy`.
+
+In previous versions of pandas, a single grouping with :meth:`.DataFrameGroupBy.apply` or :meth:`.DataFrameGroupBy.agg` would pass the unobserved groups to the provided function, resulting in ``0`` below.
+
+.. ipython:: python
+
+ df = pd.DataFrame(
+ {
+ "key1": pd.Categorical(list("aabb"), categories=list("abc")),
+ "key2": [1, 1, 1, 2],
+ "values": [1, 2, 3, 4],
+ }
+ )
+ df
+ gb = df.groupby("key1", observed=False)
+ gb[["values"]].apply(lambda x: x.sum())
+
+However this was not the case when using multiple groupings, resulting in ``NaN`` below.
+
+.. code-block:: ipython
+
+ In [1]: gb = df.groupby(["key1", "key2"], observed=False)
+ In [2]: gb[["values"]].apply(lambda x: x.sum())
+ Out[2]:
+ values
+ key1 key2
+ a 1 3.0
+ 2 NaN
+ b 1 3.0
+ 2 4.0
+ c 1 NaN
+ 2 NaN
+
+Now using multiple groupings will also pass the unobserved groups to the provided function.
+
+.. ipython:: python
+
+ gb = df.groupby(["key1", "key2"], observed=False)
+ gb[["values"]].apply(lambda x: x.sum())
+
+Similarly:
+
+- In previous versions of pandas the method :meth:`.DataFrameGroupBy.sum` would result in ``0`` for unobserved groups, but :meth:`.DataFrameGroupBy.prod`, :meth:`.DataFrameGroupBy.all`, and :meth:`.DataFrameGroupBy.any` would all result in NA values. Now these methods result in ``1``, ``True``, and ``False`` respectively.
+- :meth:`.DataFrameGroupBy.groups` did not include unobserved groups and now does.
+
+These improvements also fixed certain bugs in groupby:
+
+- :meth:`.DataFrameGroupBy.agg` would fail when there are multiple groupings, unobserved groups, and ``as_index=False`` (:issue:`36698`)
+- :meth:`.DataFrameGroupBy.groups` with ``sort=False`` would sort groups; they now occur in the order they are observed (:issue:`56966`)
+- :meth:`.DataFrameGroupBy.nunique` would fail when there are multiple groupings, unobserved groups, and ``as_index=False`` (:issue:`52848`)
+- :meth:`.DataFrameGroupBy.sum` would have incorrect values when there are multiple groupings, unobserved groups, and non-numeric data (:issue:`43891`)
+- :meth:`.DataFrameGroupBy.value_counts` would produce incorrect results when used with some categorical and some non-categorical groupings and ``observed=False`` (:issue:`56016`)
+
+.. _whatsnew_300.notable_bug_fixes.notable_bug_fix2:
+
+notable_bug_fix2
+^^^^^^^^^^^^^^^^
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_300.api_breaking:
+
+Backwards incompatible API changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _whatsnew_300.api_breaking.deps:
+
+Increased minimum versions for dependencies
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Some minimum supported versions of dependencies were updated.
+If installed, we now require:
+
++-----------------+-----------------+----------+---------+
+| Package | Minimum Version | Required | Changed |
++=================+=================+==========+=========+
+| numpy | 1.23.5 | X | X |
++-----------------+-----------------+----------+---------+
+
+For `optional libraries `_ the general recommendation is to use the latest version.
+The following table lists the lowest version per library that is currently being tested throughout the development of pandas.
+Optional libraries below the lowest tested version may still work, but are not considered supported.
+
++------------------------+---------------------+
+| Package | New Minimum Version |
++========================+=====================+
+| fastparquet | 2023.10.0 |
++------------------------+---------------------+
+| adbc-driver-postgresql | 0.10.0 |
++------------------------+---------------------+
+
+See :ref:`install.dependencies` and :ref:`install.optional_dependencies` for more.
+
+.. _whatsnew_300.api_breaking.other:
+
+Other API changes
+^^^^^^^^^^^^^^^^^
+- 3rd party ``py.path`` objects are no longer explicitly supported in IO methods. Use :py:class:`pathlib.Path` objects instead (:issue:`57091`)
+- :attr:`MultiIndex.codes`, :attr:`MultiIndex.levels`, and :attr:`MultiIndex.names` now returns a ``tuple`` instead of a ``FrozenList`` (:issue:`53531`)
+- :func:`read_table`'s ``parse_dates`` argument defaults to ``None`` to improve consistency with :func:`read_csv` (:issue:`57476`)
+- Made ``dtype`` a required argument in :meth:`ExtensionArray._from_sequence_of_strings` (:issue:`56519`)
+- Updated :meth:`DataFrame.to_excel` so that the output spreadsheet has no styling. Custom styling can still be done using :meth:`Styler.to_excel` (:issue:`54154`)
+- pickle and HDF (``.h5``) files created with Python 2 are no longer explicitly supported (:issue:`57387`)
+- pickled objects from pandas version less than ``1.0.0`` are no longer supported (:issue:`57155`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_300.deprecations:
+
+Deprecations
+~~~~~~~~~~~~
+
+Copy keyword
+^^^^^^^^^^^^
+
+The ``copy`` keyword argument in the following methods is deprecated and
+will be removed in a future version:
+
+- :meth:`DataFrame.truncate` / :meth:`Series.truncate`
+- :meth:`DataFrame.tz_convert` / :meth:`Series.tz_convert`
+- :meth:`DataFrame.tz_localize` / :meth:`Series.tz_localize`
+- :meth:`DataFrame.infer_objects` / :meth:`Series.infer_objects`
+- :meth:`DataFrame.align` / :meth:`Series.align`
+- :meth:`DataFrame.astype` / :meth:`Series.astype`
+- :meth:`DataFrame.reindex` / :meth:`Series.reindex`
+- :meth:`DataFrame.reindex_like` / :meth:`Series.reindex_like`
+- :meth:`DataFrame.set_axis` / :meth:`Series.set_axis`
+- :meth:`DataFrame.to_period` / :meth:`Series.to_period`
+- :meth:`DataFrame.to_timestamp` / :meth:`Series.to_timestamp`
+- :meth:`DataFrame.rename` / :meth:`Series.rename`
+- :meth:`DataFrame.transpose`
+- :meth:`DataFrame.swaplevel`
+- :meth:`DataFrame.merge` / :func:`pd.merge`
+
+Copy-on-Write utilizes a lazy copy mechanism that defers copying the data until
+necessary. Use ``.copy`` to trigger an eager copy. The copy keyword has no effect
+starting with 3.0, so it can be safely removed from your code.
+
+Other Deprecations
+^^^^^^^^^^^^^^^^^^
+
+- Deprecated :meth:`Timestamp.utcfromtimestamp`, use ``Timestamp.fromtimestamp(ts, "UTC")`` instead (:issue:`56680`)
+- Deprecated :meth:`Timestamp.utcnow`, use ``Timestamp.now("UTC")`` instead (:issue:`56680`)
+- Deprecated allowing non-keyword arguments in :meth:`Series.to_markdown` except ``buf``. (:issue:`57280`)
+- Deprecated allowing non-keyword arguments in :meth:`Series.to_string` except ``buf``. (:issue:`57280`)
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_300.prior_deprecations:
+
+Removal of prior version deprecations/changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- :class:`.DataFrameGroupBy.idxmin`, :class:`.DataFrameGroupBy.idxmax`, :class:`.SeriesGroupBy.idxmin`, and :class:`.SeriesGroupBy.idxmax` will now raise a ``ValueError`` when used with ``skipna=False`` and an NA value is encountered (:issue:`10694`)
+- :func:`concat` no longer ignores empty objects when determining output dtypes (:issue:`39122`)
+- :func:`read_excel`, :func:`read_json`, :func:`read_html`, and :func:`read_xml` no longer accept raw string or byte representation of the data. That type of data must be wrapped in a :py:class:`StringIO` or :py:class:`BytesIO` (:issue:`53767`)
+- :meth:`DataFrame.groupby` with ``as_index=False`` and aggregation methods will no longer exclude from the result the groupings that do not arise from the input (:issue:`49519`)
+- :meth:`Series.dt.to_pydatetime` now returns a :class:`Series` of :py:class:`datetime.datetime` objects (:issue:`52459`)
+- :meth:`SeriesGroupBy.agg` no longer pins the name of the group to the input passed to the provided ``func`` (:issue:`51703`)
+- All arguments except ``name`` in :meth:`Index.rename` are now keyword only (:issue:`56493`)
+- All arguments except the first ``path``-like argument in IO writers are now keyword only (:issue:`54229`)
+- Disallow non-standard (``np.ndarray``, :class:`Index`, :class:`ExtensionArray`, or :class:`Series`) to :func:`isin`, :func:`unique`, :func:`factorize` (:issue:`52986`)
+- Disallow passing a pandas type to :meth:`Index.view` (:issue:`55709`)
+- Disallow units other than "s", "ms", "us", "ns" for datetime64 and timedelta64 dtypes in :func:`array` (:issue:`53817`)
+- Removed "freq" keyword from :class:`PeriodArray` constructor, use "dtype" instead (:issue:`52462`)
+- Removed deprecated "method" and "limit" keywords from :meth:`Series.replace` and :meth:`DataFrame.replace` (:issue:`53492`)
+- Removed extension test classes ``BaseNoReduceTests``, ``BaseNumericReduceTests``, ``BaseBooleanReduceTests`` (:issue:`54663`)
+- Removed the "closed" and "normalize" keywords in :meth:`DatetimeIndex.__new__` (:issue:`52628`)
+- Stopped performing dtype inference with in :meth:`Index.insert` with object-dtype index; this often affects the index/columns that result when setting new entries into an empty :class:`Series` or :class:`DataFrame` (:issue:`51363`)
+- Removed the "closed" and "unit" keywords in :meth:`TimedeltaIndex.__new__` (:issue:`52628`, :issue:`55499`)
+- All arguments in :meth:`Index.sort_values` are now keyword only (:issue:`56493`)
+- All arguments in :meth:`Series.to_dict` are now keyword only (:issue:`56493`)
+- Changed the default value of ``observed`` in :meth:`DataFrame.groupby` and :meth:`Series.groupby` to ``True`` (:issue:`51811`)
+- Enforce deprecation in :func:`testing.assert_series_equal` and :func:`testing.assert_frame_equal` with object dtype and mismatched null-like values, which are now considered not-equal (:issue:`18463`)
+- Enforced deprecation ``all`` and ``any`` reductions with ``datetime64`` and :class:`DatetimeTZDtype` dtypes (:issue:`58029`)
+- Enforced deprecation disallowing parsing datetimes with mixed time zones unless user passes ``utc=True`` to :func:`to_datetime` (:issue:`57275`)
+- Enforced deprecation in :meth:`Series.value_counts` and :meth:`Index.value_counts` with object dtype performing dtype inference on the ``.index`` of the result (:issue:`56161`)
+- Enforced deprecation of :meth:`.DataFrameGroupBy.get_group` and :meth:`.SeriesGroupBy.get_group` allowing the ``name`` argument to be a non-tuple when grouping by a list of length 1 (:issue:`54155`)
+- Enforced deprecation of :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` for object-dtype (:issue:`57820`)
+- Enforced deprecation of :meth:`offsets.Tick.delta`, use ``pd.Timedelta(obj)`` instead (:issue:`55498`)
+- Enforced deprecation of ``axis=None`` acting the same as ``axis=0`` in the DataFrame reductions ``sum``, ``prod``, ``std``, ``var``, and ``sem``, passing ``axis=None`` will now reduce over both axes; this is particularly the case when doing e.g. ``numpy.sum(df)`` (:issue:`21597`)
+- Enforced deprecation of non-standard (``np.ndarray``, :class:`ExtensionArray`, :class:`Index`, or :class:`Series`) argument to :func:`api.extensions.take` (:issue:`52981`)
+- Enforced deprecation of parsing system timezone strings to ``tzlocal``, which depended on system timezone, pass the 'tz' keyword instead (:issue:`50791`)
+- Enforced deprecation of passing a dictionary to :meth:`SeriesGroupBy.agg` (:issue:`52268`)
+- Enforced deprecation of string ``AS`` denoting frequency in :class:`YearBegin` and strings ``AS-DEC``, ``AS-JAN``, etc. denoting annual frequencies with various fiscal year starts (:issue:`57793`)
+- Enforced deprecation of string ``A`` denoting frequency in :class:`YearEnd` and strings ``A-DEC``, ``A-JAN``, etc. denoting annual frequencies with various fiscal year ends (:issue:`57699`)
+- Enforced deprecation of string ``BAS`` denoting frequency in :class:`BYearBegin` and strings ``BAS-DEC``, ``BAS-JAN``, etc. denoting annual frequencies with various fiscal year starts (:issue:`57793`)
+- Enforced deprecation of string ``BA`` denoting frequency in :class:`BYearEnd` and strings ``BA-DEC``, ``BA-JAN``, etc. denoting annual frequencies with various fiscal year ends (:issue:`57793`)
+- Enforced deprecation of strings ``T``, ``L``, ``U``, and ``N`` denoting frequencies in :class:`Minute`, :class:`Second`, :class:`Milli`, :class:`Micro`, :class:`Nano` (:issue:`57627`)
+- Enforced deprecation of strings ``T``, ``L``, ``U``, and ``N`` denoting units in :class:`Timedelta` (:issue:`57627`)
+- Enforced deprecation of the behavior of :func:`concat` when ``len(keys) != len(objs)`` would truncate to the shorter of the two. Now this raises a ``ValueError`` (:issue:`43485`)
+- Enforced deprecation of values "pad", "ffill", "bfill", and "backfill" for :meth:`Series.interpolate` and :meth:`DataFrame.interpolate` (:issue:`57869`)
+- Enforced deprecation removing :meth:`Categorical.to_list`, use ``obj.tolist()`` instead (:issue:`51254`)
+- Enforced silent-downcasting deprecation for :ref:`all relevant methods ` (:issue:`54710`)
+- In :meth:`DataFrame.stack`, the default value of ``future_stack`` is now ``True``; specifying ``False`` will raise a ``FutureWarning`` (:issue:`55448`)
+- Iterating over a :class:`.DataFrameGroupBy` or :class:`.SeriesGroupBy` will return tuples of length 1 for the groups when grouping by ``level`` a list of length 1 (:issue:`50064`)
+- Methods ``apply``, ``agg``, and ``transform`` will no longer replace NumPy functions (e.g. ``np.sum``) and built-in functions (e.g. ``min``) with the equivalent pandas implementation; use string aliases (e.g. ``"sum"`` and ``"min"``) if you desire to use the pandas implementation (:issue:`53974`)
+- Passing both ``freq`` and ``fill_value`` in :meth:`DataFrame.shift` and :meth:`Series.shift` and :meth:`.DataFrameGroupBy.shift` now raises a ``ValueError`` (:issue:`54818`)
+- Removed :meth:`.DataFrameGroupBy.quantile` and :meth:`.SeriesGroupBy.quantile` supporting bool dtype (:issue:`53975`)
+- Removed :meth:`DateOffset.is_anchored` and :meth:`offsets.Tick.is_anchored` (:issue:`56594`)
+- Removed ``DataFrame.applymap``, ``Styler.applymap`` and ``Styler.applymap_index`` (:issue:`52364`)
+- Removed ``DataFrame.bool`` and ``Series.bool`` (:issue:`51756`)
+- Removed ``DataFrame.first`` and ``DataFrame.last`` (:issue:`53710`)
+- Removed ``DataFrame.swapaxes`` and ``Series.swapaxes`` (:issue:`51946`)
+- Removed ``DataFrameGroupBy.grouper`` and ``SeriesGroupBy.grouper`` (:issue:`56521`)
+- Removed ``DataFrameGroupby.fillna`` and ``SeriesGroupBy.fillna``` (:issue:`55719`)
+- Removed ``Index.format``, use :meth:`Index.astype` with ``str`` or :meth:`Index.map` with a ``formatter`` function instead (:issue:`55439`)
+- Removed ``Resample.fillna`` (:issue:`55719`)
+- Removed ``Series.__int__`` and ``Series.__float__``. Call ``int(Series.iloc[0])`` or ``float(Series.iloc[0])`` instead. (:issue:`51131`)
+- Removed ``Series.ravel`` (:issue:`56053`)
+- Removed ``Series.view`` (:issue:`56054`)
+- Removed ``StataReader.close`` (:issue:`49228`)
+- Removed ``_data`` from :class:`DataFrame`, :class:`Series`, :class:`.arrays.ArrowExtensionArray` (:issue:`52003`)
+- Removed ``axis`` argument from :meth:`DataFrame.groupby`, :meth:`Series.groupby`, :meth:`DataFrame.rolling`, :meth:`Series.rolling`, :meth:`DataFrame.resample`, and :meth:`Series.resample` (:issue:`51203`)
+- Removed ``axis`` argument from all groupby operations (:issue:`50405`)
+- Removed ``convert_dtype`` from :meth:`Series.apply` (:issue:`52257`)
+- Removed ``method``, ``limit`` ``fill_axis`` and ``broadcast_axis`` keywords from :meth:`DataFrame.align` (:issue:`51968`)
+- Removed ``pandas.api.types.is_interval`` and ``pandas.api.types.is_period``, use ``isinstance(obj, pd.Interval)`` and ``isinstance(obj, pd.Period)`` instead (:issue:`55264`)
+- Removed ``pandas.io.sql.execute`` (:issue:`50185`)
+- Removed ``pandas.value_counts``, use :meth:`Series.value_counts` instead (:issue:`53493`)
+- Removed ``read_gbq`` and ``DataFrame.to_gbq``. Use ``pandas_gbq.read_gbq`` and ``pandas_gbq.to_gbq`` instead https://pandas-gbq.readthedocs.io/en/latest/api.html (:issue:`55525`)
+- Removed ``use_nullable_dtypes`` from :func:`read_parquet` (:issue:`51853`)
+- Removed ``year``, ``month``, ``quarter``, ``day``, ``hour``, ``minute``, and ``second`` keywords in the :class:`PeriodIndex` constructor, use :meth:`PeriodIndex.from_fields` instead (:issue:`55960`)
+- Removed argument ``limit`` from :meth:`DataFrame.pct_change`, :meth:`Series.pct_change`, :meth:`.DataFrameGroupBy.pct_change`, and :meth:`.SeriesGroupBy.pct_change`; the argument ``method`` must be set to ``None`` and will be removed in a future version of pandas (:issue:`53520`)
+- Removed deprecated argument ``obj`` in :meth:`.DataFrameGroupBy.get_group` and :meth:`.SeriesGroupBy.get_group` (:issue:`53545`)
+- Removed deprecated behavior of :meth:`Series.agg` using :meth:`Series.apply` (:issue:`53325`)
+- Removed deprecated keyword ``method`` on :meth:`Series.fillna`, :meth:`DataFrame.fillna` (:issue:`57760`)
+- Removed option ``mode.use_inf_as_na``, convert inf entries to ``NaN`` before instead (:issue:`51684`)
+- Removed support for :class:`DataFrame` in :meth:`DataFrame.from_records`(:issue:`51697`)
+- Removed support for ``errors="ignore"`` in :func:`to_datetime`, :func:`to_timedelta` and :func:`to_numeric` (:issue:`55734`)
+- Removed support for ``slice`` in :meth:`DataFrame.take` (:issue:`51539`)
+- Removed the ``ArrayManager`` (:issue:`55043`)
+- Removed the ``fastpath`` argument from the :class:`Series` constructor (:issue:`55466`)
+- Removed the ``is_boolean``, ``is_integer``, ``is_floating``, ``holds_integer``, ``is_numeric``, ``is_categorical``, ``is_object``, and ``is_interval`` attributes of :class:`Index` (:issue:`50042`)
+- Removed the ``ordinal`` keyword in :class:`PeriodIndex`, use :meth:`PeriodIndex.from_ordinals` instead (:issue:`55960`)
+- Removed unused arguments ``*args`` and ``**kwargs`` in :class:`Resampler` methods (:issue:`50977`)
+- Unrecognized timezones when parsing strings to datetimes now raises a ``ValueError`` (:issue:`51477`)
+- Removed the :class:`Grouper` attributes ``ax``, ``groups``, ``indexer``, and ``obj`` (:issue:`51206`, :issue:`51182`)
+- Removed deprecated keyword ``verbose`` on :func:`read_csv` and :func:`read_table` (:issue:`56556`)
+- Removed the ``method`` keyword in ``ExtensionArray.fillna``, implement ``ExtensionArray._pad_or_backfill`` instead (:issue:`53621`)
+- Removed the attribute ``dtypes`` from :class:`.DataFrameGroupBy` (:issue:`51997`)
+- Enforced deprecation of ``argmin``, ``argmax``, ``idxmin``, and ``idxmax`` returning a result when ``skipna=False`` and an NA value is encountered or all values are NA values; these operations will now raise in such cases (:issue:`33941`, :issue:`51276`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_300.performance:
+
+Performance improvements
+~~~~~~~~~~~~~~~~~~~~~~~~
+- :attr:`Categorical.categories` returns a :class:`RangeIndex` columns instead of an :class:`Index` if the constructed ``values`` was a ``range``. (:issue:`57787`)
+- :class:`DataFrame` returns a :class:`RangeIndex` columns when possible when ``data`` is a ``dict`` (:issue:`57943`)
+- :func:`concat` returns a :class:`RangeIndex` level in the :class:`MultiIndex` result when ``keys`` is a ``range`` or :class:`RangeIndex` (:issue:`57542`)
+- :meth:`RangeIndex.append` returns a :class:`RangeIndex` instead of a :class:`Index` when appending values that could continue the :class:`RangeIndex` (:issue:`57467`)
+- :meth:`Series.str.extract` returns a :class:`RangeIndex` columns instead of an :class:`Index` column when possible (:issue:`57542`)
+- :meth:`Series.str.partition` with :class:`ArrowDtype` returns a :class:`RangeIndex` columns instead of an :class:`Index` column when possible (:issue:`57768`)
+- Performance improvement in :class:`DataFrame` when ``data`` is a ``dict`` and ``columns`` is specified (:issue:`24368`)
+- Performance improvement in :meth:`DataFrame.join` for sorted but non-unique indexes (:issue:`56941`)
+- Performance improvement in :meth:`DataFrame.join` when left and/or right are non-unique and ``how`` is ``"left"``, ``"right"``, or ``"inner"`` (:issue:`56817`)
+- Performance improvement in :meth:`DataFrame.join` with ``how="left"`` or ``how="right"`` and ``sort=True`` (:issue:`56919`)
+- Performance improvement in :meth:`DataFrameGroupBy.ffill`, :meth:`DataFrameGroupBy.bfill`, :meth:`SeriesGroupBy.ffill`, and :meth:`SeriesGroupBy.bfill` (:issue:`56902`)
+- Performance improvement in :meth:`Index.join` by propagating cached attributes in cases where the result matches one of the inputs (:issue:`57023`)
+- Performance improvement in :meth:`Index.take` when ``indices`` is a full range indexer from zero to length of index (:issue:`56806`)
+- Performance improvement in :meth:`Index.to_frame` returning a :class:`RangeIndex` columns of a :class:`Index` when possible. (:issue:`58018`)
+- Performance improvement in :meth:`MultiIndex.equals` for equal length indexes (:issue:`56990`)
+- Performance improvement in :meth:`RangeIndex.__getitem__` with a boolean mask or integers returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57588`)
+- Performance improvement in :meth:`RangeIndex.append` when appending the same index (:issue:`57252`)
+- Performance improvement in :meth:`RangeIndex.argmin` and :meth:`RangeIndex.argmax` (:issue:`57823`)
+- Performance improvement in :meth:`RangeIndex.insert` returning a :class:`RangeIndex` instead of a :class:`Index` when the :class:`RangeIndex` is empty. (:issue:`57833`)
+- Performance improvement in :meth:`RangeIndex.round` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57824`)
+- Performance improvement in :meth:`RangeIndex.join` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57651`, :issue:`57752`)
+- Performance improvement in :meth:`RangeIndex.reindex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57647`, :issue:`57752`)
+- Performance improvement in :meth:`RangeIndex.take` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57445`, :issue:`57752`)
+- Performance improvement in :func:`merge` if hash-join can be used (:issue:`57970`)
+- Performance improvement in ``DataFrameGroupBy.__len__`` and ``SeriesGroupBy.__len__`` (:issue:`57595`)
+- Performance improvement in indexing operations for string dtypes (:issue:`56997`)
+- Performance improvement in unary methods on a :class:`RangeIndex` returning a :class:`RangeIndex` instead of a :class:`Index` when possible. (:issue:`57825`)
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_300.bug_fixes:
+
+Bug fixes
+~~~~~~~~~
+- Fixed bug in :class:`SparseDtype` for equal comparison with na fill value. (:issue:`54770`)
+- Fixed bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`)
+- Fixed bug in :meth:`DataFrame.cumsum` which was raising ``IndexError`` if dtype is ``timedelta64[ns]`` (:issue:`57956`)
+- Fixed bug in :meth:`DataFrame.join` inconsistently setting result index name (:issue:`55815`)
+- Fixed bug in :meth:`DataFrame.to_string` that raised ``StopIteration`` with nested DataFrames. (:issue:`16098`)
+- Fixed bug in :meth:`DataFrame.transform` that was returning the wrong order unless the index was monotonically increasing. (:issue:`57069`)
+- Fixed bug in :meth:`DataFrame.update` bool dtype being converted to object (:issue:`55509`)
+- Fixed bug in :meth:`DataFrameGroupBy.apply` that was returning a completely empty DataFrame when all return values of ``func`` were ``None`` instead of returning an empty DataFrame with the original columns and dtypes. (:issue:`57775`)
+- Fixed bug in :meth:`Series.diff` allowing non-integer values for the ``periods`` argument. (:issue:`56607`)
+- Fixed bug in :meth:`Series.rank` that doesn't preserve missing values for nullable integers when ``na_option='keep'``. (:issue:`56976`)
+- Fixed bug in :meth:`Series.replace` and :meth:`DataFrame.replace` inconsistently replacing matching instances when ``regex=True`` and missing values are present. (:issue:`56599`)
+
+Categorical
+^^^^^^^^^^^
+-
+-
+
+Datetimelike
+^^^^^^^^^^^^
+- Bug in :func:`date_range` where the last valid timestamp would sometimes not be produced (:issue:`56134`)
+- Bug in :func:`date_range` where using a negative frequency value would not include all points between the start and end values (:issue:`56382`)
+-
+
+Timedelta
+^^^^^^^^^
+- Accuracy improvement in :meth:`Timedelta.to_pytimedelta` to round microseconds consistently for large nanosecond based Timedelta (:issue:`57841`)
+-
+
+Timezones
+^^^^^^^^^
+-
+-
+
+Numeric
+^^^^^^^
+- Bug in ``np.matmul`` with :class:`Index` inputs raising a ``TypeError`` (:issue:`57079`)
+-
+
+Conversion
+^^^^^^^^^^
+- Bug in :meth:`Series.astype` might modify read-only array inplace when casting to a string dtype (:issue:`57212`)
+- Bug in :meth:`Series.reindex` not maintaining ``float32`` type when a ``reindex`` introduces a missing value (:issue:`45857`)
+
+Strings
+^^^^^^^
+- Bug in :meth:`Series.value_counts` would not respect ``sort=False`` for series having ``string`` dtype (:issue:`55224`)
+-
+
+Interval
+^^^^^^^^
+- Bug in :func:`interval_range` where start and end numeric types were always cast to 64 bit (:issue:`57268`)
+-
+
+Indexing
+^^^^^^^^
+-
+-
+
+Missing
+^^^^^^^
+-
+-
+
+MultiIndex
+^^^^^^^^^^
+-
+-
+
+I/O
+^^^
+- Bug in :meth:`DataFrame.to_excel` when writing empty :class:`DataFrame` with :class:`MultiIndex` on both axes (:issue:`57696`)
+- Now all ``Mapping`` s are pretty printed correctly. Before only literal ``dict`` s were. (:issue:`57915`)
+-
+-
+
+Period
+^^^^^^
+-
+-
+
+Plotting
+^^^^^^^^
+-
+-
+
+Groupby/resample/rolling
+^^^^^^^^^^^^^^^^^^^^^^^^
+- Bug in :meth:`.DataFrameGroupBy.groups` and :meth:`.SeriesGroupby.groups` that would not respect groupby argument ``dropna`` (:issue:`55919`)
+- Bug in :meth:`.DataFrameGroupBy.quantile` when ``interpolation="nearest"`` is inconsistent with :meth:`DataFrame.quantile` (:issue:`47942`)
+- Bug in :meth:`DataFrame.ewm` and :meth:`Series.ewm` when passed ``times`` and aggregation functions other than mean (:issue:`51695`)
+-
+
+Reshaping
+^^^^^^^^^
+-
+-
+
+Sparse
+^^^^^^
+-
+-
+
+ExtensionArray
+^^^^^^^^^^^^^^
+- Fixed bug in :meth:`api.types.is_datetime64_any_dtype` where a custom :class:`ExtensionDtype` would return ``False`` for array-likes (:issue:`57055`)
+-
+
+Styler
+^^^^^^
+-
+
+Other
+^^^^^
+- Bug in :class:`DataFrame` when passing a ``dict`` with a NA scalar and ``columns`` that would always return ``np.nan`` (:issue:`57205`)
+- Bug in :func:`tseries.api.guess_datetime_format` would fail to infer time format when "%Y" == "%H%M" (:issue:`57452`)
+- Bug in :func:`unique` on :class:`Index` not always returning :class:`Index` (:issue:`57043`)
+- Bug in :meth:`DataFrame.sort_index` when passing ``axis="columns"`` and ``ignore_index=True`` and ``ascending=False`` not returning a :class:`RangeIndex` columns (:issue:`57293`)
+- Bug in :meth:`DataFrame.where` where using a non-bool type array in the function would return a ``ValueError`` instead of a ``TypeError`` (:issue:`56330`)
+- Bug in Dataframe Interchange Protocol implementation was returning incorrect results for data buffers' associated dtype, for string and datetime columns (:issue:`54781`)
+
+.. ***DO NOT USE THIS SECTION***
+
+-
+-
+
+.. ---------------------------------------------------------------------------
+.. _whatsnew_300.contributors:
+
+Contributors
+~~~~~~~~~~~~
diff --git a/doc/sphinxext/announce.py b/doc/sphinxext/announce.py
index 89361eb75606c..66e999e251c5e 100755
--- a/doc/sphinxext/announce.py
+++ b/doc/sphinxext/announce.py
@@ -32,6 +32,7 @@
$ ./scripts/announce.py $GITHUB v1.11.0..v1.11.1 > announce.rst
"""
+
import codecs
import os
import re
diff --git a/doc/sphinxext/contributors.py b/doc/sphinxext/contributors.py
index c2b21e40cadad..06f205b5cc3ce 100644
--- a/doc/sphinxext/contributors.py
+++ b/doc/sphinxext/contributors.py
@@ -14,6 +14,7 @@
While the v0.23.1 tag does not exist, that will use the HEAD of the
branch as the end of the revision range.
"""
+
from announce import build_components
from docutils import nodes
from docutils.parsers.rst import Directive
diff --git a/environment.yml b/environment.yml
index 58eb69ad1f070..186d7e1d703df 100644
--- a/environment.yml
+++ b/environment.yml
@@ -8,7 +8,7 @@ dependencies:
# build dependencies
- versioneer[toml]
- - cython=3.0.5
+ - cython~=3.0.5
- meson[ninja]=1.2.1
- meson-python=0.13.1
@@ -17,6 +17,7 @@ dependencies:
- pytest-cov
- pytest-xdist>=2.2.0
- pytest-qt>=4.2.0
+ - pytest-localserver
- pyqt>=5.15.9
- coverage
@@ -29,7 +30,7 @@ dependencies:
- beautifulsoup4>=4.11.2
- blosc
- bottleneck>=1.3.6
- - fastparquet>=2022.12.0
+ - fastparquet>=2023.10.0
- fsspec>=2022.11.0
- html5lib>=1.1
- hypothesis>=6.46.1
@@ -115,8 +116,7 @@ dependencies:
- pygments # Code highlighting
- pip:
- - adbc-driver-postgresql>=0.8.0
+ - adbc-driver-postgresql>=0.10.0
- adbc-driver-sqlite>=0.8.0
- - dataframe-api-compat>=0.1.7
- typing_extensions; python_version<"3.11"
- tzdata>=2022.7
diff --git a/pandas/__init__.py b/pandas/__init__.py
index 7fab662ed2de4..3ee6f6abf97bf 100644
--- a/pandas/__init__.py
+++ b/pandas/__init__.py
@@ -1,8 +1,5 @@
from __future__ import annotations
-import os
-import warnings
-
__docformat__ = "restructuredtext"
# Let users know if they're missing any of our hard dependencies
@@ -31,7 +28,8 @@
raise ImportError(
f"C extension: {_module} not built. If you want to import "
"pandas from the source directory, you may need to run "
- "'python setup.py build_ext' to build the C extensions first."
+ "'python -m pip install -ve . --no-build-isolation --config-settings "
+ "editable-verbose=true' to build the C extensions first."
) from _err
from pandas._config import (
@@ -102,7 +100,6 @@
Grouper,
factorize,
unique,
- value_counts,
NamedAgg,
array,
Categorical,
@@ -163,7 +160,6 @@
read_parquet,
read_orc,
read_feather,
- read_gbq,
read_html,
read_xml,
read_json,
@@ -193,17 +189,6 @@
__git_version__ = v.get("full-revisionid")
del get_versions, v
-# GH#55043 - deprecation of the data_manager option
-if "PANDAS_DATA_MANAGER" in os.environ:
- warnings.warn(
- "The env variable PANDAS_DATA_MANAGER is set. The data_manager option is "
- "deprecated and will be removed in a future version. Only the BlockManager "
- "will be available. Unset this environment variable to silence this warning.",
- FutureWarning,
- stacklevel=2,
- )
-# Don't allow users to use pandas.os or pandas.warnings
-del os, warnings
# module level doc-string
__doc__ = """
@@ -334,7 +319,6 @@
"read_excel",
"read_feather",
"read_fwf",
- "read_gbq",
"read_hdf",
"read_html",
"read_json",
@@ -362,6 +346,5 @@
"to_timedelta",
"tseries",
"unique",
- "value_counts",
"wide_to_long",
]
diff --git a/pandas/_config/__init__.py b/pandas/_config/__init__.py
index 97784c924dab4..e1999dd536999 100644
--- a/pandas/_config/__init__.py
+++ b/pandas/_config/__init__.py
@@ -5,6 +5,7 @@
importing `dates` and `display` ensures that keys needed by _libs
are initialized.
"""
+
__all__ = [
"config",
"detect_console_encoding",
@@ -15,7 +16,6 @@
"option_context",
"options",
"using_copy_on_write",
- "warn_copy_on_write",
]
from pandas._config import config
from pandas._config import dates # pyright: ignore[reportUnusedImport] # noqa: F401
@@ -32,24 +32,7 @@
def using_copy_on_write() -> bool:
- _mode_options = _global_config["mode"]
- return (
- _mode_options["copy_on_write"] is True
- and _mode_options["data_manager"] == "block"
- )
-
-
-def warn_copy_on_write() -> bool:
- _mode_options = _global_config["mode"]
- return (
- _mode_options["copy_on_write"] == "warn"
- and _mode_options["data_manager"] == "block"
- )
-
-
-def using_nullable_dtypes() -> bool:
- _mode_options = _global_config["mode"]
- return _mode_options["nullable_dtypes"]
+ return True
def using_pyarrow_string_dtype() -> bool:
diff --git a/pandas/_config/config.py b/pandas/_config/config.py
index 73d69105541d8..8921e1b686303 100644
--- a/pandas/_config/config.py
+++ b/pandas/_config/config.py
@@ -50,31 +50,23 @@
from __future__ import annotations
-from contextlib import (
- ContextDecorator,
- contextmanager,
-)
+from contextlib import contextmanager
import re
from typing import (
TYPE_CHECKING,
Any,
Callable,
- Generic,
NamedTuple,
cast,
)
import warnings
-from pandas._typing import (
- F,
- T,
-)
+from pandas._typing import F
from pandas.util._exceptions import find_stack_level
if TYPE_CHECKING:
from collections.abc import (
Generator,
- Iterable,
Sequence,
)
@@ -88,7 +80,7 @@ class DeprecatedOption(NamedTuple):
class RegisteredOption(NamedTuple):
key: str
- defval: object
+ defval: Any
doc: str
validator: Callable[[object], Any] | None
cb: Callable[[str], Any] | None
@@ -125,68 +117,168 @@ class OptionError(AttributeError, KeyError):
# User API
-def _get_single_key(pat: str, silent: bool) -> str:
+def _get_single_key(pat: str) -> str:
keys = _select_options(pat)
if len(keys) == 0:
- if not silent:
- _warn_if_deprecated(pat)
- raise OptionError(f"No such keys(s): {repr(pat)}")
+ _warn_if_deprecated(pat)
+ raise OptionError(f"No such keys(s): {pat!r}")
if len(keys) > 1:
raise OptionError("Pattern matched multiple keys")
key = keys[0]
- if not silent:
- _warn_if_deprecated(key)
+ _warn_if_deprecated(key)
key = _translate_key(key)
return key
-def _get_option(pat: str, silent: bool = False) -> Any:
- key = _get_single_key(pat, silent)
+def get_option(pat: str) -> Any:
+ """
+ Retrieve the value of the specified option.
+
+ Parameters
+ ----------
+ pat : str
+ Regexp which should match a single option.
+
+ .. warning::
+
+ Partial matches are supported for convenience, but unless you use the
+ full option name (e.g. x.y.z.option_name), your code may break in future
+ versions if new options with similar names are introduced.
+
+ Returns
+ -------
+ Any
+ The value of the option.
+
+ Raises
+ ------
+ OptionError : if no such option exists
+
+ Notes
+ -----
+ For all available options, please view the :ref:`User Guide `
+ or use ``pandas.describe_option()``.
+
+ Examples
+ --------
+ >>> pd.get_option("display.max_columns") # doctest: +SKIP
+ 4
+ """
+ key = _get_single_key(pat)
# walk the nested dict
root, k = _get_root(key)
return root[k]
-def _set_option(*args, **kwargs) -> None:
+def set_option(*args) -> None:
+ """
+ Set the value of the specified option or options.
+
+ Parameters
+ ----------
+ *args : str | object
+ Arguments provided in pairs, which will be interpreted as (pattern, value)
+ pairs.
+ pattern: str
+ Regexp which should match a single option
+ value: object
+ New value of option
+
+ .. warning::
+
+ Partial pattern matches are supported for convenience, but unless you
+ use the full option name (e.g. x.y.z.option_name), your code may break in
+ future versions if new options with similar names are introduced.
+
+ Returns
+ -------
+ None
+ No return value.
+
+ Raises
+ ------
+ ValueError if odd numbers of non-keyword arguments are provided
+ TypeError if keyword arguments are provided
+ OptionError if no such option exists
+
+ Notes
+ -----
+ For all available options, please view the :ref:`User Guide `
+ or use ``pandas.describe_option()``.
+
+ Examples
+ --------
+ >>> pd.set_option("display.max_columns", 4)
+ >>> df = pd.DataFrame([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]])
+ >>> df
+ 0 1 ... 3 4
+ 0 1 2 ... 4 5
+ 1 6 7 ... 9 10
+ [2 rows x 5 columns]
+ >>> pd.reset_option("display.max_columns")
+ """
# must at least 1 arg deal with constraints later
nargs = len(args)
if not nargs or nargs % 2 != 0:
raise ValueError("Must provide an even number of non-keyword arguments")
- # default to false
- silent = kwargs.pop("silent", False)
-
- if kwargs:
- kwarg = next(iter(kwargs.keys()))
- raise TypeError(f'_set_option() got an unexpected keyword argument "{kwarg}"')
-
for k, v in zip(args[::2], args[1::2]):
- key = _get_single_key(k, silent)
+ key = _get_single_key(k)
- o = _get_registered_option(key)
- if o and o.validator:
- o.validator(v)
+ opt = _get_registered_option(key)
+ if opt and opt.validator:
+ opt.validator(v)
# walk the nested dict
root, k_root = _get_root(key)
root[k_root] = v
- if o.cb:
- if silent:
- with warnings.catch_warnings(record=True):
- o.cb(key)
- else:
- o.cb(key)
+ if opt.cb:
+ opt.cb(key)
+
+
+def describe_option(pat: str = "", _print_desc: bool = True) -> str | None:
+ """
+ Print the description for one or more registered options.
+ Call with no arguments to get a listing for all registered options.
-def _describe_option(pat: str = "", _print_desc: bool = True) -> str | None:
+ Parameters
+ ----------
+ pat : str, default ""
+ String or string regexp pattern.
+ Empty string will return all options.
+ For regexp strings, all matching keys will have their description displayed.
+ _print_desc : bool, default True
+ If True (default) the description(s) will be printed to stdout.
+ Otherwise, the description(s) will be returned as a string
+ (for testing).
+
+ Returns
+ -------
+ None
+ If ``_print_desc=True``.
+ str
+ If the description(s) as a string if ``_print_desc=False``.
+
+ Notes
+ -----
+ For all available options, please view the
+ :ref:`User Guide `.
+
+ Examples
+ --------
+ >>> pd.describe_option("display.max_columns") # doctest: +SKIP
+ display.max_columns : int
+ If max_cols is exceeded, switch to truncate view...
+ """
keys = _select_options(pat)
if len(keys) == 0:
- raise OptionError("No such keys(s)")
+ raise OptionError(f"No such keys(s) for {pat=}")
s = "\n".join([_build_option_description(k) for k in keys])
@@ -196,11 +288,40 @@ def _describe_option(pat: str = "", _print_desc: bool = True) -> str | None:
return s
-def _reset_option(pat: str, silent: bool = False) -> None:
+def reset_option(pat: str) -> None:
+ """
+ Reset one or more options to their default value.
+
+ Parameters
+ ----------
+ pat : str/regex
+ If specified only options matching ``pat*`` will be reset.
+ Pass ``"all"`` as argument to reset all options.
+
+ .. warning::
+
+ Partial matches are supported for convenience, but unless you
+ use the full option name (e.g. x.y.z.option_name), your code may break
+ in future versions if new options with similar names are introduced.
+
+ Returns
+ -------
+ None
+ No return value.
+
+ Notes
+ -----
+ For all available options, please view the
+ :ref:`User Guide `.
+
+ Examples
+ --------
+ >>> pd.reset_option("display.max_columns") # doctest: +SKIP
+ """
keys = _select_options(pat)
if len(keys) == 0:
- raise OptionError("No such keys(s)")
+ raise OptionError(f"No such keys(s) for {pat=}")
if len(keys) > 1 and len(pat) < 4 and pat != "all":
raise ValueError(
@@ -210,11 +331,11 @@ def _reset_option(pat: str, silent: bool = False) -> None:
)
for k in keys:
- _set_option(k, _registered_options[k].defval, silent=silent)
+ set_option(k, _registered_options[k].defval)
def get_default_val(pat: str):
- key = _get_single_key(pat, silent=True)
+ key = _get_single_key(pat)
return _get_registered_option(key).defval
@@ -235,7 +356,7 @@ def __setattr__(self, key: str, val: Any) -> None:
# you can't set new keys
# can you can't overwrite subtrees
if key in self.d and not isinstance(self.d[key], dict):
- _set_option(prefix, val)
+ set_option(prefix, val)
else:
raise OptionError("You can only set the value of existing options")
@@ -251,242 +372,60 @@ def __getattr__(self, key: str):
if isinstance(v, dict):
return DictWrapper(v, prefix)
else:
- return _get_option(prefix)
+ return get_option(prefix)
def __dir__(self) -> list[str]:
return list(self.d.keys())
-# For user convenience, we'd like to have the available options described
-# in the docstring. For dev convenience we'd like to generate the docstrings
-# dynamically instead of maintaining them by hand. To this, we use the
-# class below which wraps functions inside a callable, and converts
-# __doc__ into a property function. The doctsrings below are templates
-# using the py2.6+ advanced formatting syntax to plug in a concise list
-# of options, and option descriptions.
-
-
-class CallableDynamicDoc(Generic[T]):
- def __init__(self, func: Callable[..., T], doc_tmpl: str) -> None:
- self.__doc_tmpl__ = doc_tmpl
- self.__func__ = func
-
- def __call__(self, *args, **kwds) -> T:
- return self.__func__(*args, **kwds)
-
- # error: Signature of "__doc__" incompatible with supertype "object"
- @property
- def __doc__(self) -> str: # type: ignore[override]
- opts_desc = _describe_option("all", _print_desc=False)
- opts_list = pp_options_list(list(_registered_options.keys()))
- return self.__doc_tmpl__.format(opts_desc=opts_desc, opts_list=opts_list)
-
-
-_get_option_tmpl = """
-get_option(pat)
-
-Retrieves the value of the specified option.
-
-Available options:
-
-{opts_list}
-
-Parameters
-----------
-pat : str
- Regexp which should match a single option.
- Note: partial matches are supported for convenience, but unless you use the
- full option name (e.g. x.y.z.option_name), your code may break in future
- versions if new options with similar names are introduced.
-
-Returns
--------
-result : the value of the option
-
-Raises
-------
-OptionError : if no such option exists
-
-Notes
------
-Please reference the :ref:`User Guide ` for more information.
-
-The available options with its descriptions:
-
-{opts_desc}
-
-Examples
---------
->>> pd.get_option('display.max_columns') # doctest: +SKIP
-4
-"""
-
-_set_option_tmpl = """
-set_option(pat, value)
-
-Sets the value of the specified option.
-
-Available options:
-
-{opts_list}
-
-Parameters
-----------
-pat : str
- Regexp which should match a single option.
- Note: partial matches are supported for convenience, but unless you use the
- full option name (e.g. x.y.z.option_name), your code may break in future
- versions if new options with similar names are introduced.
-value : object
- New value of option.
-
-Returns
--------
-None
-
-Raises
-------
-OptionError if no such option exists
-
-Notes
------
-Please reference the :ref:`User Guide ` for more information.
-
-The available options with its descriptions:
-
-{opts_desc}
-
-Examples
---------
->>> pd.set_option('display.max_columns', 4)
->>> df = pd.DataFrame([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]])
->>> df
- 0 1 ... 3 4
-0 1 2 ... 4 5
-1 6 7 ... 9 10
-[2 rows x 5 columns]
->>> pd.reset_option('display.max_columns')
-"""
-
-_describe_option_tmpl = """
-describe_option(pat, _print_desc=False)
-
-Prints the description for one or more registered options.
-
-Call with no arguments to get a listing for all registered options.
-
-Available options:
-
-{opts_list}
-
-Parameters
-----------
-pat : str
- Regexp pattern. All matching keys will have their description displayed.
-_print_desc : bool, default True
- If True (default) the description(s) will be printed to stdout.
- Otherwise, the description(s) will be returned as a unicode string
- (for testing).
-
-Returns
--------
-None by default, the description(s) as a unicode string if _print_desc
-is False
-
-Notes
------
-Please reference the :ref:`User Guide ` for more information.
-
-The available options with its descriptions:
-
-{opts_desc}
-
-Examples
---------
->>> pd.describe_option('display.max_columns') # doctest: +SKIP
-display.max_columns : int
- If max_cols is exceeded, switch to truncate view...
-"""
-
-_reset_option_tmpl = """
-reset_option(pat)
-
-Reset one or more options to their default value.
-
-Pass "all" as argument to reset all options.
-
-Available options:
-
-{opts_list}
-
-Parameters
-----------
-pat : str/regex
- If specified only options matching `prefix*` will be reset.
- Note: partial matches are supported for convenience, but unless you
- use the full option name (e.g. x.y.z.option_name), your code may break
- in future versions if new options with similar names are introduced.
-
-Returns
--------
-None
-
-Notes
------
-Please reference the :ref:`User Guide ` for more information.
-
-The available options with its descriptions:
-
-{opts_desc}
-
-Examples
---------
->>> pd.reset_option('display.max_columns') # doctest: +SKIP
-"""
-
-# bind the functions with their docstrings into a Callable
-# and use that as the functions exposed in pd.api
-get_option = CallableDynamicDoc(_get_option, _get_option_tmpl)
-set_option = CallableDynamicDoc(_set_option, _set_option_tmpl)
-reset_option = CallableDynamicDoc(_reset_option, _reset_option_tmpl)
-describe_option = CallableDynamicDoc(_describe_option, _describe_option_tmpl)
options = DictWrapper(_global_config)
#
# Functions for use by pandas developers, in addition to User - api
-class option_context(ContextDecorator):
+@contextmanager
+def option_context(*args) -> Generator[None, None, None]:
"""
- Context manager to temporarily set options in the `with` statement context.
+ Context manager to temporarily set options in a ``with`` statement.
- You need to invoke as ``option_context(pat, val, [(pat, val), ...])``.
+ Parameters
+ ----------
+ *args : str | object
+ An even amount of arguments provided in pairs which will be
+ interpreted as (pattern, value) pairs.
+
+ Returns
+ -------
+ None
+ No return value.
+
+ Notes
+ -----
+ For all available options, please view the :ref:`User Guide `
+ or use ``pandas.describe_option()``.
Examples
--------
>>> from pandas import option_context
- >>> with option_context('display.max_rows', 10, 'display.max_columns', 5):
+ >>> with option_context("display.max_rows", 10, "display.max_columns", 5):
... pass
"""
+ if len(args) % 2 != 0 or len(args) < 2:
+ raise ValueError(
+ "Provide an even amount of arguments as "
+ "option_context(pat, val, pat, val...)."
+ )
- def __init__(self, *args) -> None:
- if len(args) % 2 != 0 or len(args) < 2:
- raise ValueError(
- "Need to invoke as option_context(pat, val, [(pat, val), ...])."
- )
-
- self.ops = list(zip(args[::2], args[1::2]))
-
- def __enter__(self) -> None:
- self.undo = [(pat, _get_option(pat)) for pat, val in self.ops]
-
- for pat, val in self.ops:
- _set_option(pat, val, silent=True)
-
- def __exit__(self, *args) -> None:
- if self.undo:
- for pat, val in self.undo:
- _set_option(pat, val, silent=True)
+ ops = tuple(zip(args[::2], args[1::2]))
+ try:
+ undo = tuple((pat, get_option(pat)) for pat, val in ops)
+ for pat, val in ops:
+ set_option(pat, val)
+ yield
+ finally:
+ for pat, val in undo:
+ set_option(pat, val)
def register_option(
@@ -641,12 +580,6 @@ def _get_root(key: str) -> tuple[dict[str, Any], str]:
return cursor, path[-1]
-def _is_deprecated(key: str) -> bool:
- """Returns True if the given option has been deprecated"""
- key = key.lower()
- return key in _deprecated_options
-
-
def _get_deprecated_option(key: str):
"""
Retrieves the metadata for a deprecated option, if `key` is deprecated.
@@ -729,7 +662,10 @@ def _build_option_description(k: str) -> str:
s += "No description available."
if o:
- s += f"\n [default: {o.defval}] [currently: {_get_option(k, True)}]"
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", FutureWarning)
+ warnings.simplefilter("ignore", DeprecationWarning)
+ s += f"\n [default: {o.defval}] [currently: {get_option(k)}]"
if d:
rkey = d.rkey or ""
@@ -740,41 +676,6 @@ def _build_option_description(k: str) -> str:
return s
-def pp_options_list(keys: Iterable[str], width: int = 80, _print: bool = False):
- """Builds a concise listing of available options, grouped by prefix"""
- from itertools import groupby
- from textwrap import wrap
-
- def pp(name: str, ks: Iterable[str]) -> list[str]:
- pfx = "- " + name + ".[" if name else ""
- ls = wrap(
- ", ".join(ks),
- width,
- initial_indent=pfx,
- subsequent_indent=" ",
- break_long_words=False,
- )
- if ls and ls[-1] and name:
- ls[-1] = ls[-1] + "]"
- return ls
-
- ls: list[str] = []
- singles = [x for x in sorted(keys) if x.find(".") < 0]
- if singles:
- ls += pp("", singles)
- keys = [x for x in keys if x.find(".") >= 0]
-
- for k, g in groupby(sorted(keys), lambda x: x[: x.rfind(".")]):
- ks = [x[len(k) + 1 :] for x in list(g)]
- ls += pp(k, ks)
- s = "\n".join(ls)
- if _print:
- print(s)
- else:
- return s
-
-
-#
# helpers
diff --git a/pandas/_config/dates.py b/pandas/_config/dates.py
index b37831f96eb73..2d9f5d390dc9c 100644
--- a/pandas/_config/dates.py
+++ b/pandas/_config/dates.py
@@ -1,6 +1,7 @@
"""
config for datetime formatting
"""
+
from __future__ import annotations
from pandas._config import config as cf
diff --git a/pandas/_config/localization.py b/pandas/_config/localization.py
index 5c1a0ff139533..61d88c43f0e4a 100644
--- a/pandas/_config/localization.py
+++ b/pandas/_config/localization.py
@@ -3,6 +3,7 @@
Name `localization` is chosen to avoid overlap with builtin `locale` module.
"""
+
from __future__ import annotations
from contextlib import contextmanager
@@ -10,7 +11,10 @@
import platform
import re
import subprocess
-from typing import TYPE_CHECKING
+from typing import (
+ TYPE_CHECKING,
+ cast,
+)
from pandas._config.config import options
@@ -152,7 +156,7 @@ def get_locales(
out_locales = []
for x in split_raw_locales:
try:
- out_locales.append(str(x, encoding=options.display.encoding))
+ out_locales.append(str(x, encoding=cast(str, options.display.encoding)))
except UnicodeError:
# 'locale -a' is used to populated 'raw_locales' and on
# Redhat 7 Linux (and maybe others) prints locale names
diff --git a/pandas/_libs/algos.pyi b/pandas/_libs/algos.pyi
index caf5425dfc7b4..0a6be851e1efd 100644
--- a/pandas/_libs/algos.pyi
+++ b/pandas/_libs/algos.pyi
@@ -165,6 +165,15 @@ def take_1d_int32_float64(
def take_1d_int64_int64(
values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
) -> None: ...
+def take_1d_uint16_uint16(
+ values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
+) -> None: ...
+def take_1d_uint32_uint32(
+ values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
+) -> None: ...
+def take_1d_uint64_uint64(
+ values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
+) -> None: ...
def take_1d_int64_float64(
values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
) -> None: ...
@@ -225,6 +234,15 @@ def take_2d_axis0_int64_int64(
def take_2d_axis0_int64_float64(
values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
) -> None: ...
+def take_2d_axis0_uint16_uint16(
+ values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
+) -> None: ...
+def take_2d_axis0_uint32_uint32(
+ values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
+) -> None: ...
+def take_2d_axis0_uint64_uint64(
+ values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
+) -> None: ...
def take_2d_axis0_float32_float32(
values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
) -> None: ...
@@ -279,6 +297,15 @@ def take_2d_axis1_int32_float64(
def take_2d_axis1_int64_int64(
values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
) -> None: ...
+def take_2d_axis1_uint16_uint16(
+ values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
+) -> None: ...
+def take_2d_axis1_uint32_uint32(
+ values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
+) -> None: ...
+def take_2d_axis1_uint64_uint64(
+ values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
+) -> None: ...
def take_2d_axis1_int64_float64(
values: np.ndarray, indexer: npt.NDArray[np.intp], out: np.ndarray, fill_value=...
) -> None: ...
diff --git a/pandas/_libs/algos.pyx b/pandas/_libs/algos.pyx
index e70ac26a2c28e..e2e93c5242b24 100644
--- a/pandas/_libs/algos.pyx
+++ b/pandas/_libs/algos.pyx
@@ -180,6 +180,8 @@ def is_lexsorted(list_of_arrays: list) -> bool:
n = len(list_of_arrays[0])
cdef int64_t **vecs = malloc(nlevels * sizeof(int64_t*))
+ if vecs is NULL:
+ raise MemoryError()
for i in range(nlevels):
arr = list_of_arrays[i]
assert arr.dtype.name == "int64"
diff --git a/pandas/_libs/algos_take_helper.pxi.in b/pandas/_libs/algos_take_helper.pxi.in
index 88c3abba506a3..e6b39896280b9 100644
--- a/pandas/_libs/algos_take_helper.pxi.in
+++ b/pandas/_libs/algos_take_helper.pxi.in
@@ -15,6 +15,9 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
dtypes = [
('uint8_t', 'uint8_t'),
('uint8_t', 'object'),
+ ('uint16_t', 'uint16_t'),
+ ('uint32_t', 'uint32_t'),
+ ('uint64_t', 'uint64_t'),
('int8_t', 'int8_t'),
('int8_t', 'int32_t'),
('int8_t', 'int64_t'),
@@ -184,6 +187,17 @@ def take_2d_axis1_{{name}}_{{dest}}(ndarray[{{c_type_in}}, ndim=2] values,
fv = fill_value
+ {{if c_type_in == c_type_out != "object"}}
+ with nogil:
+ for i in range(n):
+ for j in range(k):
+ idx = indexer[j]
+ if idx == -1:
+ out[i, j] = fv
+ else:
+ out[i, j] = values[i, idx]
+
+ {{else}}
for i in range(n):
for j in range(k):
idx = indexer[j]
@@ -195,6 +209,7 @@ def take_2d_axis1_{{name}}_{{dest}}(ndarray[{{c_type_in}}, ndim=2] values,
{{else}}
out[i, j] = values[i, idx]
{{endif}}
+ {{endif}}
@cython.wraparound(False)
diff --git a/pandas/_libs/arrays.pyi b/pandas/_libs/arrays.pyi
index 86f69c3cdfc75..dda23d9dec98b 100644
--- a/pandas/_libs/arrays.pyi
+++ b/pandas/_libs/arrays.pyi
@@ -14,9 +14,9 @@ class NDArrayBacked:
_ndarray: np.ndarray
def __init__(self, values: np.ndarray, dtype: DtypeObj) -> None: ...
@classmethod
- def _simple_new(cls, values: np.ndarray, dtype: DtypeObj): ...
- def _from_backing_data(self, values: np.ndarray): ...
- def __setstate__(self, state): ...
+ def _simple_new(cls, values: np.ndarray, dtype: DtypeObj) -> Self: ...
+ def _from_backing_data(self, values: np.ndarray) -> Self: ...
+ def __setstate__(self, state) -> None: ...
def __len__(self) -> int: ...
@property
def shape(self) -> Shape: ...
@@ -26,14 +26,14 @@ class NDArrayBacked:
def size(self) -> int: ...
@property
def nbytes(self) -> int: ...
- def copy(self, order=...): ...
- def delete(self, loc, axis=...): ...
- def swapaxes(self, axis1, axis2): ...
- def repeat(self, repeats: int | Sequence[int], axis: int | None = ...): ...
- def reshape(self, *args, **kwargs): ...
- def ravel(self, order=...): ...
+ def copy(self, order=...) -> Self: ...
+ def delete(self, loc, axis=...) -> Self: ...
+ def swapaxes(self, axis1, axis2) -> Self: ...
+ def repeat(self, repeats: int | Sequence[int], axis: int | None = ...) -> Self: ...
+ def reshape(self, *args, **kwargs) -> Self: ...
+ def ravel(self, order=...) -> Self: ...
@property
- def T(self): ...
+ def T(self) -> Self: ...
@classmethod
def _concat_same_type(
cls, to_concat: Sequence[Self], axis: AxisInt = ...
diff --git a/pandas/_libs/groupby.pyi b/pandas/_libs/groupby.pyi
index 135828a23648a..53f5f73624232 100644
--- a/pandas/_libs/groupby.pyi
+++ b/pandas/_libs/groupby.pyi
@@ -12,6 +12,7 @@ def group_median_float64(
min_count: int = ..., # Py_ssize_t
mask: np.ndarray | None = ...,
result_mask: np.ndarray | None = ...,
+ is_datetimelike: bool = ..., # bint
) -> None: ...
def group_cumprod(
out: np.ndarray, # float64_t[:, ::1]
@@ -42,10 +43,10 @@ def group_shift_indexer(
def group_fillna_indexer(
out: np.ndarray, # ndarray[intp_t]
labels: np.ndarray, # ndarray[int64_t]
- sorted_labels: npt.NDArray[np.intp],
mask: npt.NDArray[np.uint8],
limit: int, # int64_t
- dropna: bool,
+ compute_ffill: bool,
+ ngroups: int,
) -> None: ...
def group_any_all(
out: np.ndarray, # uint8_t[::1]
@@ -136,6 +137,7 @@ def group_last(
result_mask: npt.NDArray[np.bool_] | None = ...,
min_count: int = ..., # Py_ssize_t
is_datetimelike: bool = ...,
+ skipna: bool = ...,
) -> None: ...
def group_nth(
out: np.ndarray, # rank_t[:, ::1]
@@ -147,6 +149,7 @@ def group_nth(
min_count: int = ..., # int64_t
rank: int = ..., # int64_t
is_datetimelike: bool = ...,
+ skipna: bool = ...,
) -> None: ...
def group_rank(
out: np.ndarray, # float64_t[:, ::1]
diff --git a/pandas/_libs/groupby.pyx b/pandas/_libs/groupby.pyx
index 19d71b0a6fde3..c0b9ed42cb535 100644
--- a/pandas/_libs/groupby.pyx
+++ b/pandas/_libs/groupby.pyx
@@ -81,6 +81,8 @@ cdef float64_t median_linear_mask(float64_t* a, int n, uint8_t* mask) noexcept n
return NaN
tmp = malloc((n - na_count) * sizeof(float64_t))
+ if tmp is NULL:
+ raise MemoryError()
j = 0
for i in range(n):
@@ -99,7 +101,11 @@ cdef float64_t median_linear_mask(float64_t* a, int n, uint8_t* mask) noexcept n
return result
-cdef float64_t median_linear(float64_t* a, int n) noexcept nogil:
+cdef float64_t median_linear(
+ float64_t* a,
+ int n,
+ bint is_datetimelike=False
+) noexcept nogil:
cdef:
int i, j, na_count = 0
float64_t* tmp
@@ -109,21 +115,34 @@ cdef float64_t median_linear(float64_t* a, int n) noexcept nogil:
return NaN
# count NAs
- for i in range(n):
- if a[i] != a[i]:
- na_count += 1
+ if is_datetimelike:
+ for i in range(n):
+ if a[i] == NPY_NAT:
+ na_count += 1
+ else:
+ for i in range(n):
+ if a[i] != a[i]:
+ na_count += 1
if na_count:
if na_count == n:
return NaN
tmp = malloc((n - na_count) * sizeof(float64_t))
+ if tmp is NULL:
+ raise MemoryError()
j = 0
- for i in range(n):
- if a[i] == a[i]:
- tmp[j] = a[i]
- j += 1
+ if is_datetimelike:
+ for i in range(n):
+ if a[i] != NPY_NAT:
+ tmp[j] = a[i]
+ j += 1
+ else:
+ for i in range(n):
+ if a[i] == a[i]:
+ tmp[j] = a[i]
+ j += 1
a = tmp
n -= na_count
@@ -166,6 +185,7 @@ def group_median_float64(
Py_ssize_t min_count=-1,
const uint8_t[:, :] mask=None,
uint8_t[:, ::1] result_mask=None,
+ bint is_datetimelike=False,
) -> None:
"""
Only aggregates on axis=0
@@ -224,7 +244,7 @@ def group_median_float64(
ptr += _counts[0]
for j in range(ngroups):
size = _counts[j + 1]
- out[j, i] = median_linear(ptr, size)
+ out[j, i] = median_linear(ptr, size, is_datetimelike)
ptr += size
@@ -493,10 +513,10 @@ def group_shift_indexer(
def group_fillna_indexer(
ndarray[intp_t] out,
ndarray[intp_t] labels,
- ndarray[intp_t] sorted_labels,
ndarray[uint8_t] mask,
int64_t limit,
- bint dropna,
+ bint compute_ffill,
+ int ngroups,
) -> None:
"""
Indexes how to fill values forwards or backwards within a group.
@@ -508,50 +528,52 @@ def group_fillna_indexer(
labels : np.ndarray[np.intp]
Array containing unique label for each group, with its ordering
matching up to the corresponding record in `values`.
- sorted_labels : np.ndarray[np.intp]
- obtained by `np.argsort(labels, kind="mergesort")`
- values : np.ndarray[np.uint8]
- Containing the truth value of each element.
mask : np.ndarray[np.uint8]
Indicating whether a value is na or not.
- limit : Consecutive values to fill before stopping, or -1 for no limit
- dropna : Flag to indicate if NaN groups should return all NaN values
+ limit : int64_t
+ Consecutive values to fill before stopping, or -1 for no limit.
+ compute_ffill : bint
+ Whether to compute ffill or bfill.
+ ngroups : int
+ Number of groups, larger than all entries of `labels`.
Notes
-----
This method modifies the `out` parameter rather than returning an object
"""
cdef:
- Py_ssize_t i, N, idx
- intp_t curr_fill_idx=-1
- int64_t filled_vals = 0
-
- N = len(out)
+ Py_ssize_t idx, N = len(out)
+ intp_t label
+ intp_t[::1] last = -1 * np.ones(ngroups, dtype=np.intp)
+ intp_t[::1] fill_count = np.zeros(ngroups, dtype=np.intp)
# Make sure all arrays are the same size
assert N == len(labels) == len(mask)
with nogil:
- for i in range(N):
- idx = sorted_labels[i]
- if dropna and labels[idx] == -1: # nan-group gets nan-values
- curr_fill_idx = -1
+ # Can't use for loop with +/- step
+ # https://github.com/cython/cython/issues/1106
+ idx = 0 if compute_ffill else N-1
+ for _ in range(N):
+ label = labels[idx]
+ if label == -1: # na-group gets na-values
+ out[idx] = -1
elif mask[idx] == 1: # is missing
# Stop filling once we've hit the limit
- if filled_vals >= limit and limit != -1:
- curr_fill_idx = -1
- filled_vals += 1
- else: # reset items when not missing
- filled_vals = 0
- curr_fill_idx = idx
-
- out[idx] = curr_fill_idx
+ if limit != -1 and fill_count[label] >= limit:
+ out[idx] = -1
+ else:
+ out[idx] = last[label]
+ fill_count[label] += 1
+ else:
+ fill_count[label] = 0 # reset items when not missing
+ last[label] = idx
+ out[idx] = idx
- # If we move to the next group, reset
- # the fill_idx and counter
- if i == N - 1 or labels[idx] != labels[sorted_labels[i + 1]]:
- curr_fill_idx = -1
- filled_vals = 0
+ if compute_ffill:
+ idx += 1
+ else:
+ idx -= 1
@cython.boundscheck(False)
@@ -1284,7 +1306,9 @@ def group_quantile(
elif interp == INTERPOLATION_MIDPOINT:
out[i, k] = (val + next_val) / 2.0
elif interp == INTERPOLATION_NEAREST:
- if frac > .5 or (frac == .5 and q_val > .5): # Always OK?
+ if frac > .5 or (frac == .5 and idx % 2 == 1):
+ # If quantile lies in the middle of two indexes,
+ # take the even index, as np.quantile.
out[i, k] = next_val
else:
out[i, k] = val
@@ -1424,6 +1448,7 @@ def group_last(
uint8_t[:, ::1] result_mask=None,
Py_ssize_t min_count=-1,
bint is_datetimelike=False,
+ bint skipna=True,
) -> None:
"""
Only aggregates on axis=0
@@ -1458,14 +1483,19 @@ def group_last(
for j in range(K):
val = values[i, j]
- if uses_mask:
- isna_entry = mask[i, j]
- else:
- isna_entry = _treat_as_na(val, is_datetimelike)
+ if skipna:
+ if uses_mask:
+ isna_entry = mask[i, j]
+ else:
+ isna_entry = _treat_as_na(val, is_datetimelike)
+ if isna_entry:
+ continue
- if not isna_entry:
- nobs[lab, j] += 1
- resx[lab, j] = val
+ nobs[lab, j] += 1
+ resx[lab, j] = val
+
+ if uses_mask and not skipna:
+ result_mask[lab, j] = mask[i, j]
_check_below_mincount(
out, uses_mask, result_mask, ncounts, K, nobs, min_count, resx
@@ -1486,6 +1516,7 @@ def group_nth(
int64_t min_count=-1,
int64_t rank=1,
bint is_datetimelike=False,
+ bint skipna=True,
) -> None:
"""
Only aggregates on axis=0
@@ -1520,15 +1551,19 @@ def group_nth(
for j in range(K):
val = values[i, j]
- if uses_mask:
- isna_entry = mask[i, j]
- else:
- isna_entry = _treat_as_na(val, is_datetimelike)
+ if skipna:
+ if uses_mask:
+ isna_entry = mask[i, j]
+ else:
+ isna_entry = _treat_as_na(val, is_datetimelike)
+ if isna_entry:
+ continue
- if not isna_entry:
- nobs[lab, j] += 1
- if nobs[lab, j] == rank:
- resx[lab, j] = val
+ nobs[lab, j] += 1
+ if nobs[lab, j] == rank:
+ resx[lab, j] = val
+ if uses_mask and not skipna:
+ result_mask[lab, j] = mask[i, j]
_check_below_mincount(
out, uses_mask, result_mask, ncounts, K, nobs, min_count, resx
@@ -1767,6 +1802,7 @@ def group_idxmin_idxmax(
Py_ssize_t i, j, N, K, lab
numeric_object_t val
numeric_object_t[:, ::1] group_min_or_max
+ uint8_t[:, ::1] seen
bint uses_mask = mask is not None
bint isna_entry
bint compute_max = name == "idxmax"
@@ -1780,13 +1816,10 @@ def group_idxmin_idxmax(
if numeric_object_t is object:
group_min_or_max = np.empty((out).shape, dtype=object)
+ seen = np.zeros((out).shape, dtype=np.uint8)
else:
group_min_or_max = np.empty_like(out, dtype=values.dtype)
- if N > 0 and K > 0:
- # When N or K is zero, we never use group_min_or_max
- group_min_or_max[:] = _get_min_or_max(
- values[0, 0], compute_max, is_datetimelike
- )
+ seen = np.zeros_like(out, dtype=np.uint8)
# When using transform, we need a valid value for take in the case
# a category is not observed; these values will be dropped
@@ -1802,6 +1835,7 @@ def group_idxmin_idxmax(
if not skipna and out[lab, j] == -1:
# Once we've hit NA there is no going back
continue
+
val = values[i, j]
if uses_mask:
@@ -1810,10 +1844,14 @@ def group_idxmin_idxmax(
isna_entry = _treat_as_na(val, is_datetimelike)
if isna_entry:
- if not skipna:
+ if not skipna or not seen[lab, j]:
out[lab, j] = -1
else:
- if compute_max:
+ if not seen[lab, j]:
+ seen[lab, j] = True
+ group_min_or_max[lab, j] = val
+ out[lab, j] = i
+ elif compute_max:
if val > group_min_or_max[lab, j]:
group_min_or_max[lab, j] = val
out[lab, j] = i
diff --git a/pandas/_libs/hashing.pyx b/pandas/_libs/hashing.pyx
index be6958e3315e9..8b424e96973d3 100644
--- a/pandas/_libs/hashing.pyx
+++ b/pandas/_libs/hashing.pyx
@@ -68,7 +68,11 @@ def hash_object_array(
# create an array of bytes
vecs = malloc(n * sizeof(char *))
+ if vecs is NULL:
+ raise MemoryError()
lens = malloc(n * sizeof(uint64_t))
+ if lens is NULL:
+ raise MemoryError()
for i in range(n):
val = arr[i]
diff --git a/pandas/_libs/hashtable.pxd b/pandas/_libs/hashtable.pxd
index eaec9e8462450..a5a3edad63403 100644
--- a/pandas/_libs/hashtable.pxd
+++ b/pandas/_libs/hashtable.pxd
@@ -174,16 +174,16 @@ cdef class StringHashTable(HashTable):
cdef struct Int64VectorData:
int64_t *data
- Py_ssize_t n, m
+ Py_ssize_t size, capacity
cdef class Vector:
cdef bint external_view_exists
cdef class Int64Vector(Vector):
- cdef Int64VectorData *data
+ cdef Int64VectorData data
cdef ndarray ao
- cdef resize(self)
+ cdef resize(self, Py_ssize_t new_size)
cpdef ndarray to_array(self)
cdef void append(self, int64_t x) noexcept
cdef extend(self, int64_t[:] x)
diff --git a/pandas/_libs/hashtable.pyi b/pandas/_libs/hashtable.pyi
index 555ec73acd9b2..7a810a988e50e 100644
--- a/pandas/_libs/hashtable.pyi
+++ b/pandas/_libs/hashtable.pyi
@@ -2,6 +2,7 @@ from typing import (
Any,
Hashable,
Literal,
+ overload,
)
import numpy as np
@@ -15,7 +16,7 @@ def unique_label_indices(
class Factorizer:
count: int
uniques: Any
- def __init__(self, size_hint: int) -> None: ...
+ def __init__(self, size_hint: int, uses_mask: bool = False) -> None: ...
def get_count(self) -> int: ...
def factorize(
self,
@@ -24,6 +25,9 @@ class Factorizer:
na_value=...,
mask=...,
) -> npt.NDArray[np.intp]: ...
+ def hash_inner_join(
+ self, values: np.ndarray, mask=...
+ ) -> tuple[np.ndarray, np.ndarray]: ...
class ObjectFactorizer(Factorizer):
table: PyObjectHashTable
@@ -180,18 +184,33 @@ class HashTable:
na_value: object = ...,
mask=...,
) -> npt.NDArray[np.intp]: ...
+ @overload
def unique(
self,
values: np.ndarray, # np.ndarray[subclass-specific]
- return_inverse: bool = ...,
- mask=...,
- ) -> (
- tuple[
- np.ndarray, # np.ndarray[subclass-specific]
- npt.NDArray[np.intp],
- ]
- | np.ndarray
- ): ... # np.ndarray[subclass-specific]
+ *,
+ return_inverse: Literal[False] = ...,
+ mask: None = ...,
+ ) -> np.ndarray: ... # np.ndarray[subclass-specific]
+ @overload
+ def unique(
+ self,
+ values: np.ndarray, # np.ndarray[subclass-specific]
+ *,
+ return_inverse: Literal[True],
+ mask: None = ...,
+ ) -> tuple[np.ndarray, npt.NDArray[np.intp]]: ... # np.ndarray[subclass-specific]
+ @overload
+ def unique(
+ self,
+ values: np.ndarray, # np.ndarray[subclass-specific]
+ *,
+ return_inverse: Literal[False] = ...,
+ mask: npt.NDArray[np.bool_],
+ ) -> tuple[
+ np.ndarray,
+ npt.NDArray[np.bool_],
+ ]: ... # np.ndarray[subclass-specific]
def factorize(
self,
values: np.ndarray, # np.ndarray[subclass-specific]
@@ -200,6 +219,9 @@ class HashTable:
mask=...,
ignore_na: bool = True,
) -> tuple[np.ndarray, npt.NDArray[np.intp]]: ... # np.ndarray[subclass-specific]
+ def hash_inner_join(
+ self, values: np.ndarray, mask=...
+ ) -> tuple[np.ndarray, np.ndarray]: ...
class Complex128HashTable(HashTable): ...
class Complex64HashTable(HashTable): ...
diff --git a/pandas/_libs/hashtable.pyx b/pandas/_libs/hashtable.pyx
index ccac3d0b50d45..97fae1d6480ce 100644
--- a/pandas/_libs/hashtable.pyx
+++ b/pandas/_libs/hashtable.pyx
@@ -1,8 +1,4 @@
cimport cython
-from cpython.mem cimport (
- PyMem_Free,
- PyMem_Malloc,
-)
from cpython.ref cimport (
Py_INCREF,
PyObject,
@@ -11,6 +7,7 @@ from libc.stdlib cimport (
free,
malloc,
)
+from libc.string cimport memcpy
import numpy as np
@@ -73,7 +70,7 @@ cdef class Factorizer:
cdef readonly:
Py_ssize_t count
- def __cinit__(self, size_hint: int):
+ def __cinit__(self, size_hint: int, uses_mask: bool = False):
self.count = 0
def get_count(self) -> int:
@@ -82,13 +79,16 @@ cdef class Factorizer:
def factorize(self, values, na_sentinel=-1, na_value=None, mask=None) -> np.ndarray:
raise NotImplementedError
+ def hash_inner_join(self, values, mask=None):
+ raise NotImplementedError
+
cdef class ObjectFactorizer(Factorizer):
cdef public:
PyObjectHashTable table
ObjectVector uniques
- def __cinit__(self, size_hint: int):
+ def __cinit__(self, size_hint: int, uses_mask: bool = False):
self.table = PyObjectHashTable(size_hint)
self.uniques = ObjectVector()
diff --git a/pandas/_libs/hashtable_class_helper.pxi.in b/pandas/_libs/hashtable_class_helper.pxi.in
index c0723392496c1..e3a9102fec395 100644
--- a/pandas/_libs/hashtable_class_helper.pxi.in
+++ b/pandas/_libs/hashtable_class_helper.pxi.in
@@ -133,7 +133,7 @@ dtypes = [('Complex128', 'complex128', 'khcomplex128_t'),
ctypedef struct {{name}}VectorData:
{{c_type}} *data
- Py_ssize_t n, m
+ Py_ssize_t size, capacity
{{endif}}
@@ -143,8 +143,8 @@ ctypedef struct {{name}}VectorData:
cdef void append_data_{{dtype}}({{name}}VectorData *data,
{{c_type}} x) noexcept nogil:
- data.data[data.n] = x
- data.n += 1
+ data.data[data.size] = x
+ data.size += 1
{{endfor}}
@@ -163,8 +163,9 @@ ctypedef fused vector_data:
Complex64VectorData
StringVectorData
-cdef bint needs_resize(vector_data *data) noexcept nogil:
- return data.n == data.m
+
+cdef bint needs_resize(Py_ssize_t nelems, Py_ssize_t capacity) noexcept nogil:
+ return nelems >= capacity
# ----------------------------------------------------------------------
# Vector
@@ -204,98 +205,93 @@ cdef class {{name}}Vector(Vector):
# Int64Vector is the only one we need exposed for other cython files.
{{if dtype != 'int64'}}
cdef:
- {{name}}VectorData *data
+ {{name}}VectorData data
ndarray ao
{{endif}}
def __cinit__(self):
- self.data = <{{name}}VectorData *>PyMem_Malloc(
- sizeof({{name}}VectorData))
- if not self.data:
- raise MemoryError()
- self.data.n = 0
- self.data.m = _INIT_VEC_CAP
- self.ao = np.empty(self.data.m, dtype=np.{{dtype}})
+ self.data.size = 0
+ self.data.capacity = _INIT_VEC_CAP
+ self.ao = np.empty(self.data.capacity, dtype=np.{{dtype}})
self.data.data = <{{c_type}}*>self.ao.data
- cdef resize(self):
- self.data.m = max(self.data.m * 4, _INIT_VEC_CAP)
- self.ao.resize(self.data.m, refcheck=False)
+ cdef resize(self, Py_ssize_t new_size):
+ self.data.capacity = max(new_size, _INIT_VEC_CAP)
+ self.ao.resize(self.data.capacity, refcheck=False)
self.data.data = <{{c_type}}*>self.ao.data
- def __dealloc__(self):
- if self.data is not NULL:
- PyMem_Free(self.data)
- self.data = NULL
-
def __len__(self) -> int:
- return self.data.n
+ return self.data.size
cpdef ndarray to_array(self):
- if self.data.m != self.data.n:
+ if self.data.capacity != self.data.size:
if self.external_view_exists:
# should never happen
raise ValueError("should have raised on append()")
- self.ao.resize(self.data.n, refcheck=False)
- self.data.m = self.data.n
+ self.ao.resize(self.data.size, refcheck=False)
+ self.data.capacity = self.data.size
self.external_view_exists = True
return self.ao
cdef void append(self, {{c_type}} x) noexcept:
- if needs_resize(self.data):
+ if needs_resize(self.data.size, self.data.capacity):
if self.external_view_exists:
raise ValueError("external reference but "
"Vector.resize() needed")
- self.resize()
+ self.resize(self.data.capacity * 4)
- append_data_{{dtype}}(self.data, x)
+ append_data_{{dtype}}(&self.data, x)
cdef extend(self, const {{c_type}}[:] x):
- for i in range(len(x)):
- self.append(x[i])
+ cdef Py_ssize_t x_size = len(x)
+ if x_size == 0:
+ return
+
+ cdef Py_ssize_t needed_size = self.data.size + x_size
+ if needs_resize(needed_size, self.data.capacity):
+ if self.external_view_exists:
+ raise ValueError("external reference but "
+ "Vector.resize() needed")
+ self.resize(needed_size)
+
+ memcpy(self.data.data + self.data.size, &x[0], x_size * sizeof({{c_type}}))
+ self.data.size = needed_size
{{endfor}}
cdef class StringVector(Vector):
cdef:
- StringVectorData *data
+ StringVectorData data
def __cinit__(self):
- self.data = PyMem_Malloc(sizeof(StringVectorData))
- if not self.data:
- raise MemoryError()
- self.data.n = 0
- self.data.m = _INIT_VEC_CAP
- self.data.data = malloc(self.data.m * sizeof(char *))
- if not self.data.data:
+ self.data.size = 0
+ self.data.capacity = _INIT_VEC_CAP
+ self.data.data = malloc(self.data.capacity * sizeof(char *))
+ if self.data.data is NULL:
raise MemoryError()
- cdef resize(self):
+ cdef resize(self, Py_ssize_t new_size):
cdef:
char **orig_data
- Py_ssize_t i, m
+ Py_ssize_t i, orig_capacity
- m = self.data.m
- self.data.m = max(self.data.m * 4, _INIT_VEC_CAP)
+ orig_capacity = self.data.capacity
+ self.data.capacity = max(self.data.capacity * 4, _INIT_VEC_CAP)
orig_data = self.data.data
- self.data.data = malloc(self.data.m * sizeof(char *))
- if not self.data.data:
+ self.data.data = malloc(self.data.capacity * sizeof(char *))
+ if self.data.data is NULL:
raise MemoryError()
- for i in range(m):
+ for i in range(orig_capacity):
self.data.data[i] = orig_data[i]
def __dealloc__(self):
- if self.data is not NULL:
- if self.data.data is not NULL:
- free(self.data.data)
- PyMem_Free(self.data)
- self.data = NULL
+ free(self.data.data)
def __len__(self) -> int:
- return self.data.n
+ return self.data.size
cpdef ndarray[object, ndim=1] to_array(self):
cdef:
@@ -303,20 +299,20 @@ cdef class StringVector(Vector):
Py_ssize_t n
object val
- ao = np.empty(self.data.n, dtype=object)
- for i in range(self.data.n):
+ ao = np.empty(self.data.size, dtype=object)
+ for i in range(self.data.size):
val = self.data.data[i]
ao[i] = val
self.external_view_exists = True
- self.data.m = self.data.n
+ self.data.capacity = self.data.size
return ao
cdef void append(self, char *x) noexcept:
- if needs_resize(self.data):
- self.resize()
+ if needs_resize(self.data.size, self.data.capacity):
+ self.resize(self.data.capacity * 4)
- append_data_string(self.data, x)
+ append_data_string(&self.data, x)
cdef extend(self, ndarray[object] x):
for i in range(len(x)):
@@ -327,37 +323,37 @@ cdef class ObjectVector(Vector):
cdef:
PyObject **data
- Py_ssize_t n, m
+ Py_ssize_t size, capacity
ndarray ao
def __cinit__(self):
- self.n = 0
- self.m = _INIT_VEC_CAP
+ self.size = 0
+ self.capacity = _INIT_VEC_CAP
self.ao = np.empty(_INIT_VEC_CAP, dtype=object)
self.data = self.ao.data
def __len__(self) -> int:
- return self.n
+ return self.size
cdef append(self, object obj):
- if self.n == self.m:
+ if self.size == self.capacity:
if self.external_view_exists:
raise ValueError("external reference but "
"Vector.resize() needed")
- self.m = max(self.m * 2, _INIT_VEC_CAP)
- self.ao.resize(self.m, refcheck=False)
+ self.capacity = max(self.capacity * 2, _INIT_VEC_CAP)
+ self.ao.resize(self.capacity, refcheck=False)
self.data = self.ao.data
Py_INCREF(obj)
- self.data[self.n] = obj
- self.n += 1
+ self.data[self.size] = obj
+ self.size += 1
cpdef ndarray[object, ndim=1] to_array(self):
- if self.m != self.n:
+ if self.capacity != self.size:
if self.external_view_exists:
raise ValueError("should have raised on append()")
- self.ao.resize(self.n, refcheck=False)
- self.m = self.n
+ self.ao.resize(self.size, refcheck=False)
+ self.capacity = self.size
self.external_view_exists = True
return self.ao
@@ -561,6 +557,49 @@ cdef class {{name}}HashTable(HashTable):
self.table.vals[k] = i
self.na_position = na_position
+ @cython.wraparound(False)
+ @cython.boundscheck(False)
+ def hash_inner_join(self, const {{dtype}}_t[:] values, const uint8_t[:] mask = None) -> tuple[ndarray, ndarray]:
+ cdef:
+ Py_ssize_t i, n = len(values)
+ {{c_type}} val
+ khiter_t k
+ Int64Vector locs = Int64Vector()
+ Int64Vector self_locs = Int64Vector()
+ Int64VectorData *l
+ Int64VectorData *sl
+ int8_t na_position = self.na_position
+
+ l = &locs.data
+ sl = &self_locs.data
+
+ if self.uses_mask and mask is None:
+ raise NotImplementedError # pragma: no cover
+
+ with nogil:
+ for i in range(n):
+ if self.uses_mask and mask[i]:
+ if self.na_position == -1:
+ continue
+ if needs_resize(l.size, l.capacity):
+ with gil:
+ locs.resize(locs.data.capacity * 4)
+ self_locs.resize(locs.data.capacity * 4)
+ append_data_int64(l, i)
+ append_data_int64(sl, na_position)
+ else:
+ val = {{to_c_type}}(values[i])
+ k = kh_get_{{dtype}}(self.table, val)
+ if k != self.table.n_buckets:
+ if needs_resize(l.size, l.capacity):
+ with gil:
+ locs.resize(locs.data.capacity * 4)
+ self_locs.resize(locs.data.capacity * 4)
+ append_data_int64(l, i)
+ append_data_int64(sl, self.table.vals[k])
+
+ return self_locs.to_array(), locs.to_array()
+
@cython.boundscheck(False)
def lookup(self, const {{dtype}}_t[:] values, const uint8_t[:] mask = None) -> ndarray:
# -> np.ndarray[np.intp]
@@ -652,7 +691,7 @@ cdef class {{name}}HashTable(HashTable):
if return_inverse:
labels = np.empty(n, dtype=np.intp)
- ud = uniques.data
+ ud = &uniques.data
use_na_value = na_value is not None
use_mask = mask is not None
if not use_mask and use_result_mask:
@@ -662,7 +701,7 @@ cdef class {{name}}HashTable(HashTable):
raise NotImplementedError # pragma: no cover
result_mask = UInt8Vector()
- rmd = result_mask.data
+ rmd = &result_mask.data
if use_mask:
mask_values = mask.view("uint8")
@@ -700,18 +739,18 @@ cdef class {{name}}HashTable(HashTable):
continue
seen_na = True
- if needs_resize(ud):
+ if needs_resize(ud.size, ud.capacity):
with gil:
if uniques.external_view_exists:
raise ValueError("external reference to "
"uniques held, but "
"Vector.resize() needed")
- uniques.resize()
+ uniques.resize(uniques.data.capacity * 4)
if result_mask.external_view_exists:
raise ValueError("external reference to "
"result_mask held, but "
"Vector.resize() needed")
- result_mask.resize()
+ result_mask.resize(result_mask.data.capacity * 4)
append_data_{{dtype}}(ud, val)
append_data_uint8(rmd, 1)
continue
@@ -722,19 +761,19 @@ cdef class {{name}}HashTable(HashTable):
# k hasn't been seen yet
k = kh_put_{{dtype}}(self.table, val, &ret)
- if needs_resize(ud):
+ if needs_resize(ud.size, ud.capacity):
with gil:
if uniques.external_view_exists:
raise ValueError("external reference to "
"uniques held, but "
"Vector.resize() needed")
- uniques.resize()
+ uniques.resize(uniques.data.capacity * 4)
if use_result_mask:
if result_mask.external_view_exists:
raise ValueError("external reference to "
"result_mask held, but "
"Vector.resize() needed")
- result_mask.resize()
+ result_mask.resize(result_mask.data.capacity * 4)
append_data_{{dtype}}(ud, val)
if use_result_mask:
append_data_uint8(rmd, 0)
@@ -755,7 +794,7 @@ cdef class {{name}}HashTable(HashTable):
return uniques.to_array(), result_mask.to_array()
return uniques.to_array()
- def unique(self, const {{dtype}}_t[:] values, bint return_inverse=False, object mask=None):
+ def unique(self, const {{dtype}}_t[:] values, *, bint return_inverse=False, object mask=None):
"""
Calculate unique values and labels (no sorting!)
@@ -846,7 +885,7 @@ cdef class {{name}}HashTable(HashTable):
{{name}}VectorData *ud
labels = np.empty(n, dtype=np.intp)
- ud = uniques.data
+ ud = &uniques.data
with nogil:
for i in range(n):
@@ -865,9 +904,9 @@ cdef class {{name}}HashTable(HashTable):
k = kh_put_{{dtype}}(self.table, val, &ret)
self.table.vals[k] = count
- if needs_resize(ud):
+ if needs_resize(ud.size, ud.capacity):
with gil:
- uniques.resize()
+ uniques.resize(uniques.data.capacity * 4)
append_data_{{dtype}}(ud, val)
labels[i] = count
count += 1
@@ -883,8 +922,8 @@ cdef class {{name}}Factorizer(Factorizer):
{{name}}HashTable table
{{name}}Vector uniques
- def __cinit__(self, size_hint: int):
- self.table = {{name}}HashTable(size_hint)
+ def __cinit__(self, size_hint: int, uses_mask: bool = False):
+ self.table = {{name}}HashTable(size_hint, uses_mask=uses_mask)
self.uniques = {{name}}Vector()
def factorize(self, const {{c_type}}[:] values,
@@ -915,6 +954,9 @@ cdef class {{name}}Factorizer(Factorizer):
self.count = len(self.uniques)
return labels
+ def hash_inner_join(self, const {{c_type}}[:] values, const uint8_t[:] mask = None) -> tuple[np.ndarray, np.ndarray]:
+ return self.table.hash_inner_join(values, mask)
+
{{endfor}}
@@ -933,6 +975,9 @@ cdef class StringHashTable(HashTable):
kh_destroy_str(self.table)
self.table = NULL
+ def __len__(self) -> int:
+ return self.table.size
+
def sizeof(self, deep: bool = False) -> int:
overhead = 4 * sizeof(uint32_t) + 3 * sizeof(uint32_t*)
for_flags = max(1, self.table.n_buckets >> 5) * sizeof(uint32_t)
@@ -988,6 +1033,8 @@ cdef class StringHashTable(HashTable):
const char **vecs
vecs = malloc(n * sizeof(char *))
+ if vecs is NULL:
+ raise MemoryError()
for i in range(n):
val = values[i]
v = get_c_string(val)
@@ -1018,6 +1065,8 @@ cdef class StringHashTable(HashTable):
# these by-definition *must* be strings
vecs = malloc(n * sizeof(char *))
+ if vecs is NULL:
+ raise MemoryError()
for i in range(n):
val = values[i]
@@ -1054,6 +1103,8 @@ cdef class StringHashTable(HashTable):
# these by-definition *must* be strings
vecs = malloc(n * sizeof(char *))
+ if vecs is NULL:
+ raise MemoryError()
for i in range(n):
val = values[i]
@@ -1129,6 +1180,8 @@ cdef class StringHashTable(HashTable):
# assign pointers and pre-filter out missing (if ignore_na)
vecs = malloc(n * sizeof(char *))
+ if vecs is NULL:
+ raise MemoryError()
for i in range(n):
val = values[i]
@@ -1180,7 +1233,7 @@ cdef class StringHashTable(HashTable):
return uniques.to_array(), labels.base # .base -> underlying ndarray
return uniques.to_array()
- def unique(self, ndarray[object] values, bint return_inverse=False, object mask=None):
+ def unique(self, ndarray[object] values, *, bint return_inverse=False, object mask=None):
"""
Calculate unique values and labels (no sorting!)
@@ -1438,7 +1491,7 @@ cdef class PyObjectHashTable(HashTable):
return uniques.to_array(), labels.base # .base -> underlying ndarray
return uniques.to_array()
- def unique(self, ndarray[object] values, bint return_inverse=False, object mask=None):
+ def unique(self, ndarray[object] values, *, bint return_inverse=False, object mask=None):
"""
Calculate unique values and labels (no sorting!)
diff --git a/pandas/_libs/hashtable_func_helper.pxi.in b/pandas/_libs/hashtable_func_helper.pxi.in
index 336af306d410f..5500fadb73b6d 100644
--- a/pandas/_libs/hashtable_func_helper.pxi.in
+++ b/pandas/_libs/hashtable_func_helper.pxi.in
@@ -472,7 +472,7 @@ def _unique_label_indices_{{dtype}}(const {{c_type}}[:] labels) -> ndarray:
kh_{{ttype}}_t *table = kh_init_{{ttype}}()
{{name}}Vector idx = {{name}}Vector()
ndarray[{{c_type}}, ndim=1] arr
- {{name}}VectorData *ud = idx.data
+ {{name}}VectorData *ud = &idx.data
kh_resize_{{ttype}}(table, min(kh_needed_n_buckets(n), SIZE_HINT_LIMIT))
@@ -480,9 +480,9 @@ def _unique_label_indices_{{dtype}}(const {{c_type}}[:] labels) -> ndarray:
for i in range(n):
kh_put_{{ttype}}(table, labels[i], &ret)
if ret != 0:
- if needs_resize(ud):
+ if needs_resize(ud.size, ud.capacity):
with gil:
- idx.resize()
+ idx.resize(idx.data.capacity * 4)
append_data_{{ttype}}(ud, i)
kh_destroy_{{ttype}}(table)
diff --git a/pandas/_libs/include/pandas/vendored/klib/khash_python.h b/pandas/_libs/include/pandas/vendored/klib/khash_python.h
index 5a933b45d9e21..811fdd139de2c 100644
--- a/pandas/_libs/include/pandas/vendored/klib/khash_python.h
+++ b/pandas/_libs/include/pandas/vendored/klib/khash_python.h
@@ -1,6 +1,9 @@
// Licence at LICENSES/KLIB_LICENSE
+#pragma once
+
#include
+#include
#include
typedef struct {
@@ -12,20 +15,8 @@ typedef struct {
double imag;
} khcomplex128_t;
-// khash should report usage to tracemalloc
-#if PY_VERSION_HEX >= 0x03060000
-#include
-#if PY_VERSION_HEX < 0x03070000
-#define PyTraceMalloc_Track _PyTraceMalloc_Track
-#define PyTraceMalloc_Untrack _PyTraceMalloc_Untrack
-#endif
-#else
-#define PyTraceMalloc_Track(...)
-#define PyTraceMalloc_Untrack(...)
-#endif
-
static const int KHASH_TRACE_DOMAIN = 424242;
-void *traced_malloc(size_t size) {
+static void *traced_malloc(size_t size) {
void *ptr = malloc(size);
if (ptr != NULL) {
PyTraceMalloc_Track(KHASH_TRACE_DOMAIN, (uintptr_t)ptr, size);
@@ -33,7 +24,7 @@ void *traced_malloc(size_t size) {
return ptr;
}
-void *traced_calloc(size_t num, size_t size) {
+static void *traced_calloc(size_t num, size_t size) {
void *ptr = calloc(num, size);
if (ptr != NULL) {
PyTraceMalloc_Track(KHASH_TRACE_DOMAIN, (uintptr_t)ptr, num * size);
@@ -41,7 +32,7 @@ void *traced_calloc(size_t num, size_t size) {
return ptr;
}
-void *traced_realloc(void *old_ptr, size_t size) {
+static void *traced_realloc(void *old_ptr, size_t size) {
void *ptr = realloc(old_ptr, size);
if (ptr != NULL) {
if (old_ptr != ptr) {
@@ -52,7 +43,7 @@ void *traced_realloc(void *old_ptr, size_t size) {
return ptr;
}
-void traced_free(void *ptr) {
+static void traced_free(void *ptr) {
if (ptr != NULL) {
PyTraceMalloc_Untrack(KHASH_TRACE_DOMAIN, (uintptr_t)ptr);
}
diff --git a/pandas/_libs/index.pyi b/pandas/_libs/index.pyi
index 75db47bf3160e..8cd135c944dc6 100644
--- a/pandas/_libs/index.pyi
+++ b/pandas/_libs/index.pyi
@@ -2,7 +2,10 @@ import numpy as np
from pandas._typing import npt
-from pandas import MultiIndex
+from pandas import (
+ Index,
+ MultiIndex,
+)
from pandas.core.arrays import ExtensionArray
multiindex_nulls_shift: int
@@ -50,6 +53,7 @@ class UInt32Engine(IndexEngine): ...
class UInt16Engine(IndexEngine): ...
class UInt8Engine(IndexEngine): ...
class ObjectEngine(IndexEngine): ...
+class StringEngine(IndexEngine): ...
class DatetimeEngine(Int64Engine): ...
class TimedeltaEngine(DatetimeEngine): ...
class PeriodEngine(Int64Engine): ...
@@ -69,13 +73,13 @@ class MaskedUInt8Engine(MaskedIndexEngine): ...
class MaskedBoolEngine(MaskedUInt8Engine): ...
class BaseMultiIndexCodesEngine:
- levels: list[np.ndarray]
+ levels: tuple[np.ndarray]
offsets: np.ndarray # ndarray[uint64_t, ndim=1]
def __init__(
self,
- levels: list[np.ndarray], # all entries hashable
- labels: list[np.ndarray], # all entries integer-dtyped
+ levels: tuple[Index, ...], # all entries hashable
+ labels: tuple[np.ndarray], # all entries integer-dtyped
offsets: np.ndarray, # np.ndarray[np.uint64, ndim=1]
) -> None: ...
def get_indexer(self, target: npt.NDArray[np.object_]) -> npt.NDArray[np.intp]: ...
diff --git a/pandas/_libs/index.pyx b/pandas/_libs/index.pyx
index 675288e20d1f8..a700074d46ba8 100644
--- a/pandas/_libs/index.pyx
+++ b/pandas/_libs/index.pyx
@@ -98,6 +98,20 @@ cdef ndarray _get_bool_indexer(ndarray values, object val, ndarray mask = None):
return indexer.view(bool)
+cdef _maybe_resize_array(ndarray values, Py_ssize_t loc, Py_ssize_t max_length):
+ """
+ Resize array if loc is out of bounds.
+ """
+ cdef:
+ Py_ssize_t n = len(values)
+
+ if loc >= n:
+ while loc >= n:
+ n *= 2
+ values = np.resize(values, min(n, max_length))
+ return values
+
+
# Don't populate hash tables in monotonic indexes larger than this
_SIZE_CUTOFF = 1_000_000
@@ -287,7 +301,7 @@ cdef class IndexEngine:
values = self.values
self.monotonic_inc, self.monotonic_dec, is_strict_monotonic = \
self._call_monotonic(values)
- except (TypeError, InvalidOperation):
+ except (TypeError, InvalidOperation, ValueError):
self.monotonic_inc = 0
self.monotonic_dec = 0
is_strict_monotonic = 0
@@ -456,27 +470,18 @@ cdef class IndexEngine:
# found
if val in d:
key = val
-
+ result = _maybe_resize_array(
+ result,
+ count + len(d[key]) - 1,
+ max_alloc
+ )
for j in d[key]:
-
- # realloc if needed
- if count >= n_alloc:
- n_alloc *= 2
- if n_alloc > max_alloc:
- n_alloc = max_alloc
- result = np.resize(result, n_alloc)
-
result[count] = j
count += 1
# value not found
else:
-
- if count >= n_alloc:
- n_alloc *= 2
- if n_alloc > max_alloc:
- n_alloc = max_alloc
- result = np.resize(result, n_alloc)
+ result = _maybe_resize_array(result, count, max_alloc)
result[count] = -1
count += 1
missing[count_missing] = i
@@ -533,6 +538,17 @@ cdef class ObjectEngine(IndexEngine):
return loc
+cdef class StringEngine(IndexEngine):
+
+ cdef _make_hash_table(self, Py_ssize_t n):
+ return _hash.StringHashTable(n)
+
+ cdef _check_type(self, object val):
+ if not isinstance(val, str):
+ raise KeyError(val)
+ return str(val)
+
+
cdef class DatetimeEngine(Int64Engine):
cdef:
@@ -1203,13 +1219,12 @@ cdef class MaskedIndexEngine(IndexEngine):
if PySequence_GetItem(target_mask, i):
if na_pos:
+ result = _maybe_resize_array(
+ result,
+ count + len(na_pos) - 1,
+ max_alloc,
+ )
for na_idx in na_pos:
- # realloc if needed
- if count >= n_alloc:
- n_alloc *= 2
- if n_alloc > max_alloc:
- n_alloc = max_alloc
-
result[count] = na_idx
count += 1
continue
@@ -1217,23 +1232,18 @@ cdef class MaskedIndexEngine(IndexEngine):
elif val in d:
# found
key = val
-
+ result = _maybe_resize_array(
+ result,
+ count + len(d[key]) - 1,
+ max_alloc,
+ )
for j in d[key]:
-
- # realloc if needed
- if count >= n_alloc:
- n_alloc *= 2
- if n_alloc > max_alloc:
- n_alloc = max_alloc
-
result[count] = j
count += 1
continue
# value not found
- if count >= n_alloc:
- n_alloc += 10_000
- result = np.resize(result, n_alloc)
+ result = _maybe_resize_array(result, count, max_alloc)
result[count] = -1
count += 1
missing[count_missing] = i
diff --git a/pandas/_libs/intervaltree.pxi.in b/pandas/_libs/intervaltree.pxi.in
index a6cec0fb30ecc..b94f60c272e5d 100644
--- a/pandas/_libs/intervaltree.pxi.in
+++ b/pandas/_libs/intervaltree.pxi.in
@@ -145,12 +145,12 @@ cdef class IntervalTree(IntervalMixin):
# overflow -> no match, which is already handled below
pass
- if result.data.n == old_len:
+ if result.data.size == old_len:
result.append(-1)
- elif result.data.n > old_len + 1:
+ elif result.data.size > old_len + 1:
raise KeyError(
'indexer does not intersect a unique set of intervals')
- old_len = result.data.n
+ old_len = result.data.size
return result.to_array().astype('intp')
def get_indexer_non_unique(self, ndarray[scalar_t, ndim=1] target):
@@ -172,10 +172,10 @@ cdef class IntervalTree(IntervalMixin):
# overflow -> no match, which is already handled below
pass
- if result.data.n == old_len:
+ if result.data.size == old_len:
result.append(-1)
missing.append(i)
- old_len = result.data.n
+ old_len = result.data.size
return (result.to_array().astype('intp'),
missing.to_array().astype('intp'))
diff --git a/pandas/_libs/lib.pyi b/pandas/_libs/lib.pyi
index b9fd970e68f5b..b39d32d069619 100644
--- a/pandas/_libs/lib.pyi
+++ b/pandas/_libs/lib.pyi
@@ -14,8 +14,6 @@ from typing import (
import numpy as np
-from pandas._libs.interval import Interval
-from pandas._libs.tslibs import Period
from pandas._typing import (
ArrayLike,
DtypeObj,
@@ -44,8 +42,6 @@ def is_iterator(obj: object) -> bool: ...
def is_scalar(val: object) -> bool: ...
def is_list_like(obj: object, allow_sets: bool = ...) -> bool: ...
def is_pyarrow_array(obj: object) -> bool: ...
-def is_period(val: object) -> TypeGuard[Period]: ...
-def is_interval(obj: object) -> TypeGuard[Interval]: ...
def is_decimal(obj: object) -> TypeGuard[Decimal]: ...
def is_complex(obj: object) -> TypeGuard[complex]: ...
def is_bool(obj: object) -> TypeGuard[bool | np.bool_]: ...
@@ -69,16 +65,26 @@ def fast_multiget(
mapping: dict,
keys: np.ndarray, # object[:]
default=...,
-) -> np.ndarray: ...
+) -> ArrayLike: ...
def fast_unique_multiple_list_gen(gen: Generator, sort: bool = ...) -> list: ...
def fast_unique_multiple_list(lists: list, sort: bool | None = ...) -> list: ...
+@overload
def map_infer(
arr: np.ndarray,
f: Callable[[Any], Any],
- convert: bool = ...,
+ *,
+ convert: Literal[False],
ignore_na: bool = ...,
) -> np.ndarray: ...
@overload
+def map_infer(
+ arr: np.ndarray,
+ f: Callable[[Any], Any],
+ *,
+ convert: bool = ...,
+ ignore_na: bool = ...,
+) -> ArrayLike: ...
+@overload
def maybe_convert_objects(
objects: npt.NDArray[np.object_],
*,
@@ -164,14 +170,26 @@ def is_all_arraylike(obj: list) -> bool: ...
# Functions which in reality take memoryviews
def memory_usage_of_objects(arr: np.ndarray) -> int: ... # object[:] # np.int64
+@overload
def map_infer_mask(
arr: np.ndarray,
f: Callable[[Any], Any],
mask: np.ndarray, # const uint8_t[:]
- convert: bool = ...,
+ *,
+ convert: Literal[False],
na_value: Any = ...,
dtype: np.dtype = ...,
) -> np.ndarray: ...
+@overload
+def map_infer_mask(
+ arr: np.ndarray,
+ f: Callable[[Any], Any],
+ mask: np.ndarray, # const uint8_t[:]
+ *,
+ convert: bool = ...,
+ na_value: Any = ...,
+ dtype: np.dtype = ...,
+) -> ArrayLike: ...
def indices_fast(
index: npt.NDArray[np.intp],
labels: np.ndarray, # const int64_t[:]
@@ -179,7 +197,8 @@ def indices_fast(
sorted_labels: list[npt.NDArray[np.int64]],
) -> dict[Hashable, npt.NDArray[np.intp]]: ...
def generate_slices(
- labels: np.ndarray, ngroups: int # const intp_t[:]
+ labels: np.ndarray,
+ ngroups: int, # const intp_t[:]
) -> tuple[npt.NDArray[np.int64], npt.NDArray[np.int64]]: ...
def count_level_2d(
mask: np.ndarray, # ndarray[uint8_t, ndim=2, cast=True],
@@ -209,5 +228,10 @@ def get_reverse_indexer(
def is_bool_list(obj: list) -> bool: ...
def dtypes_all_equal(types: list[DtypeObj]) -> bool: ...
def is_range_indexer(
- left: np.ndarray, n: int # np.ndarray[np.int64, ndim=1]
+ left: np.ndarray,
+ n: int, # np.ndarray[np.int64, ndim=1]
+) -> bool: ...
+def is_sequence_range(
+ sequence: np.ndarray,
+ step: int, # np.ndarray[np.int64, ndim=1]
) -> bool: ...
diff --git a/pandas/_libs/lib.pyx b/pandas/_libs/lib.pyx
index c483f35513a40..a2205454a5a46 100644
--- a/pandas/_libs/lib.pyx
+++ b/pandas/_libs/lib.pyx
@@ -678,6 +678,28 @@ def is_range_indexer(ndarray[int6432_t, ndim=1] left, Py_ssize_t n) -> bool:
return True
+@cython.wraparound(False)
+@cython.boundscheck(False)
+def is_sequence_range(ndarray[int6432_t, ndim=1] sequence, int64_t step) -> bool:
+ """
+ Check if sequence is equivalent to a range with the specified step.
+ """
+ cdef:
+ Py_ssize_t i, n = len(sequence)
+ int6432_t first_element
+
+ if step == 0:
+ return False
+ if n == 0:
+ return True
+
+ first_element = sequence[0]
+ for i in range(1, n):
+ if sequence[i] != first_element + i * step:
+ return False
+ return True
+
+
ctypedef fused ndarr_object:
ndarray[object, ndim=1]
ndarray[object, ndim=2]
@@ -759,7 +781,7 @@ cpdef ndarray[object] ensure_string_array(
out = arr.astype(str).astype(object)
out[arr.isna()] = na_value
return out
- arr = arr.to_numpy()
+ arr = arr.to_numpy(dtype=object)
elif not util.is_array(arr):
arr = np.array(arr, dtype="object")
@@ -770,6 +792,9 @@ cpdef ndarray[object] ensure_string_array(
result = result.copy()
elif not copy and result is arr:
already_copied = False
+ elif not copy and not result.flags.writeable:
+ # Weird edge case where result is a view
+ already_copied = False
if issubclass(arr.dtype.type, np.str_):
# short-circuit, all elements are str
@@ -1161,43 +1186,6 @@ cpdef bint is_decimal(object obj):
return isinstance(obj, Decimal)
-cpdef bint is_interval(object obj):
- import warnings
-
- from pandas.util._exceptions import find_stack_level
-
- warnings.warn(
- # GH#55264
- "is_interval is deprecated and will be removed in a future version. "
- "Use isinstance(obj, pd.Interval) instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return getattr(obj, "_typ", "_typ") == "interval"
-
-
-def is_period(val: object) -> bool:
- """
- Return True if given object is Period.
-
- Returns
- -------
- bool
- """
- import warnings
-
- from pandas.util._exceptions import find_stack_level
-
- warnings.warn(
- # GH#55264
- "is_period is deprecated and will be removed in a future version. "
- "Use isinstance(obj, pd.Period) instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return is_period_object(val)
-
-
def is_list_like(obj: object, allow_sets: bool = True) -> bool:
"""
Check if the object is list-like.
@@ -2864,10 +2852,11 @@ def map_infer_mask(
ndarray[object] arr,
object f,
const uint8_t[:] mask,
+ *,
bint convert=True,
object na_value=no_default,
cnp.dtype dtype=np.dtype(object)
-) -> np.ndarray:
+) -> "ArrayLike":
"""
Substitute for np.vectorize with pandas-friendly dtype inference.
@@ -2887,7 +2876,7 @@ def map_infer_mask(
Returns
-------
- np.ndarray
+ np.ndarray or an ExtensionArray
"""
cdef Py_ssize_t n = len(arr)
result = np.empty(n, dtype=dtype)
@@ -2941,8 +2930,8 @@ def _map_infer_mask(
@cython.boundscheck(False)
@cython.wraparound(False)
def map_infer(
- ndarray arr, object f, bint convert=True, bint ignore_na=False
-) -> np.ndarray:
+ ndarray arr, object f, *, bint convert=True, bint ignore_na=False
+) -> "ArrayLike":
"""
Substitute for np.vectorize with pandas-friendly dtype inference.
@@ -2956,7 +2945,7 @@ def map_infer(
Returns
-------
- np.ndarray
+ np.ndarray or an ExtensionArray
"""
cdef:
Py_ssize_t i, n
@@ -3091,7 +3080,7 @@ def to_object_array_tuples(rows: object) -> np.ndarray:
@cython.wraparound(False)
@cython.boundscheck(False)
-def fast_multiget(dict mapping, object[:] keys, default=np.nan) -> np.ndarray:
+def fast_multiget(dict mapping, object[:] keys, default=np.nan) -> "ArrayLike":
cdef:
Py_ssize_t i, n = len(keys)
object val
diff --git a/pandas/_libs/missing.pxd b/pandas/_libs/missing.pxd
index 5920649519442..899d729690451 100644
--- a/pandas/_libs/missing.pxd
+++ b/pandas/_libs/missing.pxd
@@ -7,8 +7,8 @@ from numpy cimport (
cpdef bint is_matching_na(object left, object right, bint nan_matches_none=*)
cpdef bint check_na_tuples_nonequal(object left, object right)
-cpdef bint checknull(object val, bint inf_as_na=*)
-cpdef ndarray[uint8_t] isnaobj(ndarray arr, bint inf_as_na=*)
+cpdef bint checknull(object val)
+cpdef ndarray[uint8_t] isnaobj(ndarray arr)
cdef bint is_null_datetime64(v)
cdef bint is_null_timedelta64(v)
diff --git a/pandas/_libs/missing.pyi b/pandas/_libs/missing.pyi
index 282dcee3ed6cf..6bf30a03cef32 100644
--- a/pandas/_libs/missing.pyi
+++ b/pandas/_libs/missing.pyi
@@ -11,6 +11,6 @@ def is_matching_na(
) -> bool: ...
def isposinf_scalar(val: object) -> bool: ...
def isneginf_scalar(val: object) -> bool: ...
-def checknull(val: object, inf_as_na: bool = ...) -> bool: ...
-def isnaobj(arr: np.ndarray, inf_as_na: bool = ...) -> npt.NDArray[np.bool_]: ...
+def checknull(val: object) -> bool: ...
+def isnaobj(arr: np.ndarray) -> npt.NDArray[np.bool_]: ...
def is_numeric_na(values: np.ndarray) -> npt.NDArray[np.bool_]: ...
diff --git a/pandas/_libs/missing.pyx b/pandas/_libs/missing.pyx
index 0fd1c2b21d5cd..2f44128cda822 100644
--- a/pandas/_libs/missing.pyx
+++ b/pandas/_libs/missing.pyx
@@ -137,7 +137,7 @@ cpdef bint is_matching_na(object left, object right, bint nan_matches_none=False
return False
-cpdef bint checknull(object val, bint inf_as_na=False):
+cpdef bint checknull(object val):
"""
Return boolean describing of the input is NA-like, defined here as any
of:
@@ -152,8 +152,6 @@ cpdef bint checknull(object val, bint inf_as_na=False):
Parameters
----------
val : object
- inf_as_na : bool, default False
- Whether to treat INF and -INF as NA values.
Returns
-------
@@ -164,8 +162,6 @@ cpdef bint checknull(object val, bint inf_as_na=False):
elif util.is_float_object(val) or util.is_complex_object(val):
if val != val:
return True
- elif inf_as_na:
- return val == INF or val == NEGINF
return False
elif cnp.is_timedelta64_object(val):
return cnp.get_timedelta64_value(val) == NPY_NAT
@@ -184,7 +180,7 @@ cdef bint is_decimal_na(object val):
@cython.wraparound(False)
@cython.boundscheck(False)
-cpdef ndarray[uint8_t] isnaobj(ndarray arr, bint inf_as_na=False):
+cpdef ndarray[uint8_t] isnaobj(ndarray arr):
"""
Return boolean mask denoting which elements of a 1-D array are na-like,
according to the criteria defined in `checknull`:
@@ -217,7 +213,7 @@ cpdef ndarray[uint8_t] isnaobj(ndarray arr, bint inf_as_na=False):
# equivalents to `val = values[i]`
val = cnp.PyArray_GETITEM(arr, cnp.PyArray_ITER_DATA(it))
cnp.PyArray_ITER_NEXT(it)
- is_null = checknull(val, inf_as_na=inf_as_na)
+ is_null = checknull(val)
# Dereference pointer (set value)
((cnp.PyArray_ITER_DATA(it2)))[0] = is_null
cnp.PyArray_ITER_NEXT(it2)
diff --git a/pandas/_libs/ops.pyx b/pandas/_libs/ops.pyx
index 9154e836b3477..567bfc02a2950 100644
--- a/pandas/_libs/ops.pyx
+++ b/pandas/_libs/ops.pyx
@@ -29,7 +29,7 @@ from pandas._libs.util cimport is_nan
@cython.wraparound(False)
@cython.boundscheck(False)
-def scalar_compare(object[:] values, object val, object op) -> ndarray:
+def scalar_compare(ndarray[object] values, object val, object op) -> ndarray:
"""
Compare each element of `values` array with the scalar `val`, with
the comparison operation described by `op`.
diff --git a/pandas/_libs/parsers.pyx b/pandas/_libs/parsers.pyx
index 82e9812094af2..c29cdbcf5975e 100644
--- a/pandas/_libs/parsers.pyx
+++ b/pandas/_libs/parsers.pyx
@@ -6,7 +6,6 @@ from csv import (
QUOTE_NONE,
QUOTE_NONNUMERIC,
)
-import time
import warnings
from pandas.util._exceptions import find_stack_level
@@ -344,10 +343,9 @@ cdef class TextReader:
object true_values, false_values
object handle
object orig_header
- bint na_filter, keep_default_na, verbose, has_usecols, has_mi_columns
+ bint na_filter, keep_default_na, has_usecols, has_mi_columns
bint allow_leading_cols
uint64_t parser_start # this is modified after __init__
- list clocks
const char *encoding_errors
kh_str_starts_t *false_set
kh_str_starts_t *true_set
@@ -400,7 +398,6 @@ cdef class TextReader:
bint allow_leading_cols=True,
skiprows=None,
skipfooter=0, # int64_t
- bint verbose=False,
float_precision=None,
bint skip_blank_lines=True,
encoding_errors=b"strict",
@@ -417,9 +414,6 @@ cdef class TextReader:
self.parser = parser_new()
self.parser.chunksize = tokenize_chunksize
- # For timekeeping
- self.clocks = []
-
self.parser.usecols = (usecols is not None)
self._setup_parser_source(source)
@@ -507,8 +501,6 @@ cdef class TextReader:
self.converters = converters
self.na_filter = na_filter
- self.verbose = verbose
-
if float_precision == "round_trip":
# see gh-15140
self.parser.double_converter = round_trip_wrapper
@@ -896,8 +888,6 @@ cdef class TextReader:
int64_t buffered_lines
int64_t irows
- self._start_clock()
-
if rows is not None:
irows = rows
buffered_lines = self.parser.lines - self.parser_start
@@ -915,12 +905,8 @@ cdef class TextReader:
if self.parser_start >= self.parser.lines:
raise StopIteration
- self._end_clock("Tokenization")
- self._start_clock()
columns = self._convert_column_data(rows)
- self._end_clock("Type conversion")
- self._start_clock()
if len(columns) > 0:
rows_read = len(list(columns.values())[0])
# trim
@@ -929,18 +915,8 @@ cdef class TextReader:
parser_trim_buffers(self.parser)
self.parser_start -= rows_read
- self._end_clock("Parser memory cleanup")
-
return columns
- cdef _start_clock(self):
- self.clocks.append(time.time())
-
- cdef _end_clock(self, str what):
- if self.verbose:
- elapsed = time.time() - self.clocks.pop(-1)
- print(f"{what} took: {elapsed * 1000:.2f} ms")
-
def set_noconvert(self, i: int) -> None:
self.noconvert.add(i)
@@ -1603,7 +1579,7 @@ cdef _categorical_convert(parser_t *parser, int64_t col,
# -> ndarray[f'|S{width}']
cdef _to_fw_string(parser_t *parser, int64_t col, int64_t line_start,
- int64_t line_end, int64_t width) noexcept:
+ int64_t line_end, int64_t width):
cdef:
char *data
ndarray result
diff --git a/pandas/_libs/sas.pyx b/pandas/_libs/sas.pyx
index 9e1af2cb9c3e7..209e82c6284f5 100644
--- a/pandas/_libs/sas.pyx
+++ b/pandas/_libs/sas.pyx
@@ -49,7 +49,7 @@ cdef bytes buf_as_bytes(Buffer buf, size_t offset, size_t length):
cdef Buffer buf_new(size_t length) except *:
cdef uint8_t *data = calloc(length, sizeof(uint8_t))
- if data == NULL:
+ if data is NULL:
raise MemoryError(f"Failed to allocate {length} bytes")
return Buffer(data, length)
diff --git a/pandas/_libs/src/datetime/pd_datetime.c b/pandas/_libs/src/datetime/pd_datetime.c
index 19de51be6e1b2..4c1969f6d9f57 100644
--- a/pandas/_libs/src/datetime/pd_datetime.c
+++ b/pandas/_libs/src/datetime/pd_datetime.c
@@ -20,6 +20,9 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt
#include
#include "datetime.h"
+/* Need to import_array for np_datetime.c (for NumPy 1.x support only) */
+#define PY_ARRAY_UNIQUE_SYMBOL PANDAS_DATETIME_NUMPY
+#include "numpy/ndarrayobject.h"
#include "pandas/datetime/pd_datetime.h"
#include "pandas/portable.h"
@@ -255,5 +258,6 @@ static struct PyModuleDef pandas_datetimemodule = {
PyMODINIT_FUNC PyInit_pandas_datetime(void) {
PyDateTime_IMPORT;
+ import_array();
return PyModuleDef_Init(&pandas_datetimemodule);
}
diff --git a/pandas/_libs/src/parser/tokenizer.c b/pandas/_libs/src/parser/tokenizer.c
index 0e4188bea4dc7..c9f7a796a9b1c 100644
--- a/pandas/_libs/src/parser/tokenizer.c
+++ b/pandas/_libs/src/parser/tokenizer.c
@@ -109,6 +109,14 @@ void parser_set_default_options(parser_t *self) {
parser_t *parser_new(void) { return (parser_t *)calloc(1, sizeof(parser_t)); }
+static void parser_clear_data_buffers(parser_t *self) {
+ free_if_not_null((void *)&self->stream);
+ free_if_not_null((void *)&self->words);
+ free_if_not_null((void *)&self->word_starts);
+ free_if_not_null((void *)&self->line_start);
+ free_if_not_null((void *)&self->line_fields);
+}
+
static void parser_cleanup(parser_t *self) {
// XXX where to put this
free_if_not_null((void *)&self->error_msg);
@@ -119,6 +127,7 @@ static void parser_cleanup(parser_t *self) {
self->skipset = NULL;
}
+ parser_clear_data_buffers(self);
if (self->cb_cleanup != NULL) {
self->cb_cleanup(self->source);
self->cb_cleanup = NULL;
diff --git a/pandas/_libs/src/vendored/numpy/datetime/np_datetime.c b/pandas/_libs/src/vendored/numpy/datetime/np_datetime.c
index 06e3251db8315..f854f7b9210d8 100644
--- a/pandas/_libs/src/vendored/numpy/datetime/np_datetime.c
+++ b/pandas/_libs/src/vendored/numpy/datetime/np_datetime.c
@@ -16,8 +16,6 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt
// Licence at LICENSES/NUMPY_LICENSE
-#define NO_IMPORT
-
#ifndef NPY_NO_DEPRECATED_API
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#endif // NPY_NO_DEPRECATED_API
@@ -25,7 +23,10 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt
#include
#include "pandas/vendored/numpy/datetime/np_datetime.h"
-#include
+
+#define NO_IMPORT_ARRAY
+#define PY_ARRAY_UNIQUE_SYMBOL PANDAS_DATETIME_NUMPY
+#include
#include
#if defined(_WIN32)
@@ -482,10 +483,20 @@ npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT base,
if (base == NPY_FR_ns) {
int64_t nanoseconds;
- PD_CHECK_OVERFLOW(
- scaleMicrosecondsToNanoseconds(microseconds, &nanoseconds));
- PD_CHECK_OVERFLOW(
- checked_int64_add(nanoseconds, dts->ps / 1000, &nanoseconds));
+
+ // Minimum valid timestamp in nanoseconds (1677-09-21 00:12:43.145224193).
+ const int64_t min_nanoseconds = NPY_MIN_INT64 + 1;
+ if (microseconds == min_nanoseconds / 1000 - 1) {
+ // For values within one microsecond of min_nanoseconds, use it as base
+ // and offset it with nanosecond delta to avoid overflow during scaling.
+ PD_CHECK_OVERFLOW(checked_int64_add(
+ min_nanoseconds, (dts->ps - _NS_MIN_DTS.ps) / 1000, &nanoseconds));
+ } else {
+ PD_CHECK_OVERFLOW(
+ scaleMicrosecondsToNanoseconds(microseconds, &nanoseconds));
+ PD_CHECK_OVERFLOW(
+ checked_int64_add(nanoseconds, dts->ps / 1000, &nanoseconds));
+ }
return nanoseconds;
}
@@ -699,355 +710,121 @@ void pandas_datetime_to_datetimestruct(npy_datetime dt, NPY_DATETIMEUNIT base,
void pandas_timedelta_to_timedeltastruct(npy_timedelta td,
NPY_DATETIMEUNIT base,
pandas_timedeltastruct *out) {
- npy_int64 frac;
- npy_int64 sfrac;
- npy_int64 ifrac;
- int sign;
- npy_int64 per_day;
- npy_int64 per_sec;
-
/* Initialize the output to all zeros */
memset(out, 0, sizeof(pandas_timedeltastruct));
- switch (base) {
- case NPY_FR_ns:
-
- per_day = 86400000000000LL;
- per_sec = 1000LL * 1000LL * 1000LL;
-
- // put frac in seconds
- if (td < 0 && td % per_sec != 0)
- frac = td / per_sec - 1;
- else
- frac = td / per_sec;
-
- if (frac < 0) {
- sign = -1;
-
- // even fraction
- if ((-frac % 86400LL) != 0) {
- out->days = -frac / 86400LL + 1;
- frac += 86400LL * out->days;
- } else {
- frac = -frac;
- }
- } else {
- sign = 1;
- out->days = 0;
- }
-
- if (frac >= 86400) {
- out->days += frac / 86400LL;
- frac -= out->days * 86400LL;
- }
-
- if (frac >= 3600) {
- out->hrs = (npy_int32)(frac / 3600LL);
- frac -= out->hrs * 3600LL;
- } else {
- out->hrs = 0;
- }
-
- if (frac >= 60) {
- out->min = (npy_int32)(frac / 60LL);
- frac -= out->min * 60LL;
- } else {
- out->min = 0;
- }
+ const npy_int64 sec_per_hour = 3600;
+ const npy_int64 sec_per_min = 60;
- if (frac >= 0) {
- out->sec = (npy_int32)frac;
- frac -= out->sec;
- } else {
- out->sec = 0;
- }
-
- sfrac = (out->hrs * 3600LL + out->min * 60LL + out->sec) * per_sec;
-
- if (sign < 0)
- out->days = -out->days;
-
- ifrac = td - (out->days * per_day + sfrac);
-
- if (ifrac != 0) {
- out->ms = (npy_int32)(ifrac / (1000LL * 1000LL));
- ifrac -= out->ms * 1000LL * 1000LL;
- out->us = (npy_int32)(ifrac / 1000LL);
- ifrac -= out->us * 1000LL;
- out->ns = (npy_int32)ifrac;
- } else {
- out->ms = 0;
- out->us = 0;
- out->ns = 0;
- }
+ switch (base) {
+ case NPY_FR_W:
+ out->days = 7 * td;
break;
-
- case NPY_FR_us:
-
- per_day = 86400000000LL;
- per_sec = 1000LL * 1000LL;
-
- // put frac in seconds
- if (td < 0 && td % per_sec != 0)
- frac = td / per_sec - 1;
- else
- frac = td / per_sec;
-
- if (frac < 0) {
- sign = -1;
-
- // even fraction
- if ((-frac % 86400LL) != 0) {
- out->days = -frac / 86400LL + 1;
- frac += 86400LL * out->days;
- } else {
- frac = -frac;
- }
- } else {
- sign = 1;
- out->days = 0;
- }
-
- if (frac >= 86400) {
- out->days += frac / 86400LL;
- frac -= out->days * 86400LL;
- }
-
- if (frac >= 3600) {
- out->hrs = (npy_int32)(frac / 3600LL);
- frac -= out->hrs * 3600LL;
- } else {
- out->hrs = 0;
- }
-
- if (frac >= 60) {
- out->min = (npy_int32)(frac / 60LL);
- frac -= out->min * 60LL;
- } else {
- out->min = 0;
- }
-
- if (frac >= 0) {
- out->sec = (npy_int32)frac;
- frac -= out->sec;
- } else {
- out->sec = 0;
- }
-
- sfrac = (out->hrs * 3600LL + out->min * 60LL + out->sec) * per_sec;
-
- if (sign < 0)
- out->days = -out->days;
-
- ifrac = td - (out->days * per_day + sfrac);
-
- if (ifrac != 0) {
- out->ms = (npy_int32)(ifrac / 1000LL);
- ifrac -= out->ms * 1000LL;
- out->us = (npy_int32)(ifrac / 1L);
- ifrac -= out->us * 1L;
- out->ns = (npy_int32)ifrac;
- } else {
- out->ms = 0;
- out->us = 0;
- out->ns = 0;
- }
+ case NPY_FR_D:
+ out->days = td;
break;
-
+ case NPY_FR_h:
+ out->days = td / 24LL;
+ td -= out->days * 24LL;
+ out->hrs = (npy_int32)td;
+ break;
+ case NPY_FR_m:
+ out->days = td / 1440LL;
+ td -= out->days * 1440LL;
+ out->hrs = (npy_int32)(td / 60LL);
+ td -= out->hrs * 60LL;
+ out->min = (npy_int32)td;
+ break;
+ case NPY_FR_s:
case NPY_FR_ms:
-
- per_day = 86400000LL;
- per_sec = 1000LL;
-
- // put frac in seconds
- if (td < 0 && td % per_sec != 0)
- frac = td / per_sec - 1;
- else
- frac = td / per_sec;
-
- if (frac < 0) {
- sign = -1;
-
- // even fraction
- if ((-frac % 86400LL) != 0) {
- out->days = -frac / 86400LL + 1;
- frac += 86400LL * out->days;
- } else {
- frac = -frac;
- }
- } else {
- sign = 1;
- out->days = 0;
- }
-
- if (frac >= 86400) {
- out->days += frac / 86400LL;
- frac -= out->days * 86400LL;
- }
-
- if (frac >= 3600) {
- out->hrs = (npy_int32)(frac / 3600LL);
- frac -= out->hrs * 3600LL;
- } else {
- out->hrs = 0;
- }
-
- if (frac >= 60) {
- out->min = (npy_int32)(frac / 60LL);
- frac -= out->min * 60LL;
- } else {
- out->min = 0;
- }
-
- if (frac >= 0) {
- out->sec = (npy_int32)frac;
- frac -= out->sec;
- } else {
- out->sec = 0;
- }
-
- sfrac = (out->hrs * 3600LL + out->min * 60LL + out->sec) * per_sec;
-
- if (sign < 0)
- out->days = -out->days;
-
- ifrac = td - (out->days * per_day + sfrac);
-
- if (ifrac != 0) {
- out->ms = (npy_int32)ifrac;
- out->us = 0;
- out->ns = 0;
+ case NPY_FR_us:
+ case NPY_FR_ns: {
+ const npy_int64 sec_per_day = 86400;
+ npy_int64 per_sec;
+ if (base == NPY_FR_s) {
+ per_sec = 1;
+ } else if (base == NPY_FR_ms) {
+ per_sec = 1000;
+ } else if (base == NPY_FR_us) {
+ per_sec = 1000000;
} else {
- out->ms = 0;
- out->us = 0;
- out->ns = 0;
+ per_sec = 1000000000;
}
- break;
-
- case NPY_FR_s:
- // special case where we can simplify many expressions bc per_sec=1
-
- per_day = 86400LL;
- per_sec = 1L;
+ const npy_int64 per_day = sec_per_day * per_sec;
+ npy_int64 frac;
// put frac in seconds
if (td < 0 && td % per_sec != 0)
frac = td / per_sec - 1;
else
frac = td / per_sec;
+ const int sign = frac < 0 ? -1 : 1;
if (frac < 0) {
- sign = -1;
-
// even fraction
- if ((-frac % 86400LL) != 0) {
- out->days = -frac / 86400LL + 1;
- frac += 86400LL * out->days;
+ if ((-frac % sec_per_day) != 0) {
+ out->days = -frac / sec_per_day + 1;
+ frac += sec_per_day * out->days;
} else {
frac = -frac;
}
- } else {
- sign = 1;
- out->days = 0;
}
- if (frac >= 86400) {
- out->days += frac / 86400LL;
- frac -= out->days * 86400LL;
+ if (frac >= sec_per_day) {
+ out->days += frac / sec_per_day;
+ frac -= out->days * sec_per_day;
}
- if (frac >= 3600) {
- out->hrs = (npy_int32)(frac / 3600LL);
- frac -= out->hrs * 3600LL;
- } else {
- out->hrs = 0;
+ if (frac >= sec_per_hour) {
+ out->hrs = (npy_int32)(frac / sec_per_hour);
+ frac -= out->hrs * sec_per_hour;
}
- if (frac >= 60) {
- out->min = (npy_int32)(frac / 60LL);
- frac -= out->min * 60LL;
- } else {
- out->min = 0;
+ if (frac >= sec_per_min) {
+ out->min = (npy_int32)(frac / sec_per_min);
+ frac -= out->min * sec_per_min;
}
if (frac >= 0) {
out->sec = (npy_int32)frac;
frac -= out->sec;
- } else {
- out->sec = 0;
}
- sfrac = (out->hrs * 3600LL + out->min * 60LL + out->sec) * per_sec;
-
if (sign < 0)
out->days = -out->days;
- ifrac = td - (out->days * per_day + sfrac);
-
- if (ifrac != 0) {
- out->ms = 0;
- out->us = 0;
- out->ns = 0;
- } else {
- out->ms = 0;
- out->us = 0;
- out->ns = 0;
+ if (base > NPY_FR_s) {
+ const npy_int64 sfrac =
+ (out->hrs * sec_per_hour + out->min * sec_per_min + out->sec) *
+ per_sec;
+
+ npy_int64 ifrac = td - (out->days * per_day + sfrac);
+
+ if (base == NPY_FR_ms) {
+ out->ms = (npy_int32)ifrac;
+ } else if (base == NPY_FR_us) {
+ out->ms = (npy_int32)(ifrac / 1000LL);
+ ifrac = ifrac % 1000LL;
+ out->us = (npy_int32)ifrac;
+ } else if (base == NPY_FR_ns) {
+ out->ms = (npy_int32)(ifrac / (1000LL * 1000LL));
+ ifrac = ifrac % (1000LL * 1000LL);
+ out->us = (npy_int32)(ifrac / 1000LL);
+ ifrac = ifrac % 1000LL;
+ out->ns = (npy_int32)ifrac;
+ }
}
- break;
-
- case NPY_FR_m:
-
- out->days = td / 1440LL;
- td -= out->days * 1440LL;
- out->hrs = (npy_int32)(td / 60LL);
- td -= out->hrs * 60LL;
- out->min = (npy_int32)td;
-
- out->sec = 0;
- out->ms = 0;
- out->us = 0;
- out->ns = 0;
- break;
-
- case NPY_FR_h:
- out->days = td / 24LL;
- td -= out->days * 24LL;
- out->hrs = (npy_int32)td;
-
- out->min = 0;
- out->sec = 0;
- out->ms = 0;
- out->us = 0;
- out->ns = 0;
- break;
-
- case NPY_FR_D:
- out->days = td;
- out->hrs = 0;
- out->min = 0;
- out->sec = 0;
- out->ms = 0;
- out->us = 0;
- out->ns = 0;
- break;
-
- case NPY_FR_W:
- out->days = 7 * td;
- out->hrs = 0;
- out->min = 0;
- out->sec = 0;
- out->ms = 0;
- out->us = 0;
- out->ns = 0;
- break;
+ } break;
default:
PyErr_SetString(PyExc_RuntimeError,
"NumPy timedelta metadata is corrupted with "
"invalid base unit");
+ break;
}
- out->seconds = out->hrs * 3600 + out->min * 60 + out->sec;
+ out->seconds =
+ (npy_int32)(out->hrs * sec_per_hour + out->min * sec_per_min + out->sec);
out->microseconds = out->ms * 1000 + out->us;
out->nanoseconds = out->ns;
}
@@ -1060,5 +837,8 @@ void pandas_timedelta_to_timedeltastruct(npy_timedelta td,
*/
PyArray_DatetimeMetaData
get_datetime_metadata_from_dtype(PyArray_Descr *dtype) {
- return (((PyArray_DatetimeDTypeMetaData *)dtype->c_metadata)->meta);
+#if NPY_ABI_VERSION < 0x02000000
+#define PyDataType_C_METADATA(dtype) ((dtype)->c_metadata)
+#endif
+ return ((PyArray_DatetimeDTypeMetaData *)PyDataType_C_METADATA(dtype))->meta;
}
diff --git a/pandas/_libs/src/vendored/ujson/python/objToJSON.c b/pandas/_libs/src/vendored/ujson/python/objToJSON.c
index 8bba95dd456de..fa91db5fe34e3 100644
--- a/pandas/_libs/src/vendored/ujson/python/objToJSON.c
+++ b/pandas/_libs/src/vendored/ujson/python/objToJSON.c
@@ -74,7 +74,6 @@ typedef struct __NpyArrContext {
npy_intp ndim;
npy_intp index[NPY_MAXDIMS];
int type_num;
- PyArray_GetItemFunc *getitem;
char **rowLabels;
char **columnLabels;
@@ -405,7 +404,6 @@ static void NpyArr_iterBegin(JSOBJ _obj, JSONTypeContext *tc) {
}
npyarr->array = (PyObject *)obj;
- npyarr->getitem = (PyArray_GetItemFunc *)PyArray_DESCR(obj)->f->getitem;
npyarr->dataptr = PyArray_DATA(obj);
npyarr->ndim = PyArray_NDIM(obj) - 1;
npyarr->curdim = 0;
@@ -447,8 +445,15 @@ static void NpyArrPassThru_iterEnd(JSOBJ obj, JSONTypeContext *tc) {
npyarr->curdim--;
npyarr->dataptr -= npyarr->stride * npyarr->index[npyarr->stridedim];
npyarr->stridedim -= npyarr->inc;
- npyarr->dim = PyArray_DIM(npyarr->array, npyarr->stridedim);
- npyarr->stride = PyArray_STRIDE(npyarr->array, npyarr->stridedim);
+
+ if (!PyArray_Check(npyarr->array)) {
+ PyErr_SetString(PyExc_TypeError,
+ "NpyArrayPassThru_iterEnd received a non-array object");
+ return;
+ }
+ const PyArrayObject *arrayobj = (const PyArrayObject *)npyarr->array;
+ npyarr->dim = PyArray_DIM(arrayobj, npyarr->stridedim);
+ npyarr->stride = PyArray_STRIDE(arrayobj, npyarr->stridedim);
npyarr->dataptr += npyarr->stride;
NpyArr_freeItemValue(obj, tc);
@@ -467,18 +472,25 @@ static int NpyArr_iterNextItem(JSOBJ obj, JSONTypeContext *tc) {
NpyArr_freeItemValue(obj, tc);
- if (PyArray_ISDATETIME(npyarr->array)) {
+ if (!PyArray_Check(npyarr->array)) {
+ PyErr_SetString(PyExc_TypeError,
+ "NpyArr_iterNextItem received a non-array object");
+ return 0;
+ }
+ PyArrayObject *arrayobj = (PyArrayObject *)npyarr->array;
+
+ if (PyArray_ISDATETIME(arrayobj)) {
GET_TC(tc)->itemValue = obj;
Py_INCREF(obj);
- ((PyObjectEncoder *)tc->encoder)->npyType = PyArray_TYPE(npyarr->array);
+ ((PyObjectEncoder *)tc->encoder)->npyType = PyArray_TYPE(arrayobj);
// Also write the resolution (unit) of the ndarray
- PyArray_Descr *dtype = PyArray_DESCR(npyarr->array);
+ PyArray_Descr *dtype = PyArray_DESCR(arrayobj);
((PyObjectEncoder *)tc->encoder)->valueUnit =
get_datetime_metadata_from_dtype(dtype).base;
((PyObjectEncoder *)tc->encoder)->npyValue = npyarr->dataptr;
((PyObjectEncoder *)tc->encoder)->npyCtxtPassthru = npyarr;
} else {
- GET_TC(tc)->itemValue = npyarr->getitem(npyarr->dataptr, npyarr->array);
+ GET_TC(tc)->itemValue = PyArray_GETITEM(arrayobj, npyarr->dataptr);
}
npyarr->dataptr += npyarr->stride;
@@ -505,8 +517,15 @@ static int NpyArr_iterNext(JSOBJ _obj, JSONTypeContext *tc) {
npyarr->curdim++;
npyarr->stridedim += npyarr->inc;
- npyarr->dim = PyArray_DIM(npyarr->array, npyarr->stridedim);
- npyarr->stride = PyArray_STRIDE(npyarr->array, npyarr->stridedim);
+ if (!PyArray_Check(npyarr->array)) {
+ PyErr_SetString(PyExc_TypeError,
+ "NpyArr_iterNext received a non-array object");
+ return 0;
+ }
+ const PyArrayObject *arrayobj = (const PyArrayObject *)npyarr->array;
+
+ npyarr->dim = PyArray_DIM(arrayobj, npyarr->stridedim);
+ npyarr->stride = PyArray_STRIDE(arrayobj, npyarr->stridedim);
npyarr->index[npyarr->stridedim] = 0;
((PyObjectEncoder *)tc->encoder)->npyCtxtPassthru = npyarr;
@@ -1610,7 +1629,14 @@ static void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
if (!values) {
goto INVALID;
}
- pc->columnLabelsLen = PyArray_DIM(pc->newObj, 0);
+
+ if (!PyArray_Check(pc->newObj)) {
+ PyErr_SetString(PyExc_TypeError,
+ "Object_beginTypeContext received a non-array object");
+ goto INVALID;
+ }
+ const PyArrayObject *arrayobj = (const PyArrayObject *)pc->newObj;
+ pc->columnLabelsLen = PyArray_DIM(arrayobj, 0);
pc->columnLabels = NpyArr_encodeLabels((PyArrayObject *)values, enc,
pc->columnLabelsLen);
if (!pc->columnLabels) {
diff --git a/pandas/_libs/testing.pyi b/pandas/_libs/testing.pyi
index 01da496975f51..ab87e58eba9b9 100644
--- a/pandas/_libs/testing.pyi
+++ b/pandas/_libs/testing.pyi
@@ -1,4 +1,6 @@
-def assert_dict_equal(a, b, compare_keys: bool = ...): ...
+from typing import Mapping
+
+def assert_dict_equal(a: Mapping, b: Mapping, compare_keys: bool = ...) -> bool: ...
def assert_almost_equal(
a,
b,
@@ -9,4 +11,4 @@ def assert_almost_equal(
lobj=...,
robj=...,
index_values=...,
-): ...
+) -> bool: ...
diff --git a/pandas/_libs/testing.pyx b/pandas/_libs/testing.pyx
index aed0f4b082d4e..cfd31fa610e69 100644
--- a/pandas/_libs/testing.pyx
+++ b/pandas/_libs/testing.pyx
@@ -1,6 +1,5 @@
import cmath
import math
-import warnings
import numpy as np
@@ -18,7 +17,6 @@ from pandas._libs.util cimport (
is_real_number_object,
)
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.missing import array_equivalent
@@ -188,15 +186,7 @@ cpdef assert_almost_equal(a, b,
return True
elif checknull(b):
# GH#18463
- warnings.warn(
- f"Mismatched null-like values {a} and {b} found. In a future "
- "version, pandas equality-testing functions "
- "(e.g. assert_frame_equal) will consider these not-matching "
- "and raise.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return True
+ raise AssertionError(f"Mismatched null-like values {a} != {b}")
raise AssertionError(f"{a} != {b}")
elif checknull(b):
raise AssertionError(f"{a} != {b}")
diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx
index 017fdc4bc834f..effd7f586f266 100644
--- a/pandas/_libs/tslib.pyx
+++ b/pandas/_libs/tslib.pyx
@@ -9,7 +9,6 @@ from datetime import timezone
from cpython.datetime cimport (
PyDate_Check,
PyDateTime_Check,
- datetime,
import_datetime,
timedelta,
tzinfo,
@@ -272,14 +271,13 @@ def array_with_unit_to_datetime(
"""
cdef:
Py_ssize_t i, n=len(values)
- bint is_ignore = errors == "ignore"
bint is_coerce = errors == "coerce"
bint is_raise = errors == "raise"
ndarray[int64_t] iresult
tzinfo tz = None
- float fval
+ double fval
- assert is_ignore or is_coerce or is_raise
+ assert is_coerce or is_raise
if unit == "ns":
result, tz = array_to_datetime(
@@ -342,11 +340,6 @@ def array_with_unit_to_datetime(
if is_raise:
err.args = (f"{err}, at position {i}",)
raise
- elif is_ignore:
- # we have hit an exception
- # and are in ignore mode
- # redo as object
- return _array_with_unit_to_datetime_object_fallback(values, unit)
else:
# is_coerce
iresult[i] = NPY_NAT
@@ -461,7 +454,6 @@ cpdef array_to_datetime(
bint utc_convert = bool(utc)
bint seen_datetime_offset = False
bint is_raise = errors == "raise"
- bint is_ignore = errors == "ignore"
bint is_coerce = errors == "coerce"
bint is_same_offsets
_TSObject tsobj
@@ -475,7 +467,7 @@ cpdef array_to_datetime(
str abbrev
# specify error conditions
- assert is_raise or is_ignore or is_coerce
+ assert is_raise or is_coerce
if infer_reso:
abbrev = "ns"
@@ -597,15 +589,18 @@ cpdef array_to_datetime(
return values, None
if seen_datetime_offset and not utc_convert:
- # GH#17697
+ # GH#17697, GH#57275
# 1) If all the offsets are equal, return one offset for
# the parsed dates to (maybe) pass to DatetimeIndex
- # 2) If the offsets are different, then force the parsing down the
- # object path where an array of datetimes
- # (with individual dateutil.tzoffsets) are returned
+ # 2) If the offsets are different, then do not force the parsing
+ # and raise a ValueError: "cannot parse datetimes with
+ # mixed time zones unless `utc=True`" instead
is_same_offsets = len(out_tzoffset_vals) == 1
if not is_same_offsets:
- return _array_to_datetime_object(values, errors, dayfirst, yearfirst)
+ raise ValueError(
+ "Mixed timezones detected. Pass utc=True in to_datetime "
+ "or tz='UTC' in DatetimeIndex to convert to a common timezone."
+ )
elif state.found_naive or state.found_other:
# e.g. test_to_datetime_mixed_awareness_mixed_types
raise ValueError("Cannot mix tz-aware with tz-naive values")
@@ -616,7 +611,7 @@ cpdef array_to_datetime(
if not tz_compare(tz_out, tz_out2):
# e.g. test_to_datetime_mixed_tzs_mixed_types
raise ValueError(
- "Mixed timezones detected. pass utc=True in to_datetime "
+ "Mixed timezones detected. Pass utc=True in to_datetime "
"or tz='UTC' in DatetimeIndex to convert to a common timezone."
)
# e.g. test_to_datetime_mixed_types_matching_tzs
@@ -654,116 +649,6 @@ cpdef array_to_datetime(
return result, tz_out
-@cython.wraparound(False)
-@cython.boundscheck(False)
-cdef _array_to_datetime_object(
- ndarray[object] values,
- str errors,
- bint dayfirst=False,
- bint yearfirst=False,
-):
- """
- Fall back function for array_to_datetime
-
- Attempts to parse datetime strings with dateutil to return an array
- of datetime objects
-
- Parameters
- ----------
- values : ndarray[object]
- date-like objects to convert
- errors : str
- error behavior when parsing
- dayfirst : bool, default False
- dayfirst parsing behavior when encountering datetime strings
- yearfirst : bool, default False
- yearfirst parsing behavior when encountering datetime strings
-
- Returns
- -------
- np.ndarray[object]
- Literal[None]
- """
- cdef:
- Py_ssize_t i, n = values.size
- object val
- bint is_ignore = errors == "ignore"
- bint is_coerce = errors == "coerce"
- bint is_raise = errors == "raise"
- ndarray oresult_nd
- ndarray[object] oresult
- npy_datetimestruct dts
- cnp.broadcast mi
- _TSObject tsobj
-
- assert is_raise or is_ignore or is_coerce
-
- oresult_nd = cnp.PyArray_EMPTY(values.ndim, values.shape, cnp.NPY_OBJECT, 0)
- mi = cnp.PyArray_MultiIterNew2(oresult_nd, values)
- oresult = oresult_nd.ravel()
-
- # We return an object array and only attempt to parse:
- # 1) NaT or NaT-like values
- # 2) datetime strings, which we return as datetime.datetime
- # 3) special strings - "now" & "today"
- for i in range(n):
- # Analogous to: val = values[i]
- val = (cnp.PyArray_MultiIter_DATA(mi, 1))[0]
-
- if checknull_with_nat_and_na(val) or PyDateTime_Check(val):
- # GH 25978. No need to parse NaT-like or datetime-like vals
- oresult[i] = val
- elif isinstance(val, str):
- if type(val) is not str:
- # GH#32264 np.str_ objects
- val = str(val)
-
- if len(val) == 0 or val in nat_strings:
- oresult[i] = "NaT"
- cnp.PyArray_MultiIter_NEXT(mi)
- continue
-
- try:
- tsobj = convert_str_to_tsobject(
- val, None, dayfirst=dayfirst, yearfirst=yearfirst
- )
- tsobj.ensure_reso(NPY_FR_ns, val)
-
- dts = tsobj.dts
- oresult[i] = datetime(
- dts.year, dts.month, dts.day, dts.hour, dts.min, dts.sec, dts.us,
- tzinfo=tsobj.tzinfo,
- fold=tsobj.fold,
- )
-
- except (ValueError, OverflowError) as ex:
- ex.args = (f"{ex}, at position {i}", )
- if is_coerce:
- oresult[i] = NaT
- cnp.PyArray_MultiIter_NEXT(mi)
- continue
- if is_raise:
- raise
- return values, None
- else:
- if is_raise:
- raise
- return values, None
-
- cnp.PyArray_MultiIter_NEXT(mi)
-
- warnings.warn(
- "In a future version of pandas, parsing datetimes with mixed time "
- "zones will raise an error unless `utc=True`. "
- "Please specify `utc=True` to opt in to the new behaviour "
- "and silence this warning. To create a `Series` with mixed offsets and "
- "`object` dtype, please use `apply` and `datetime.datetime.strptime`",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return oresult_nd, None
-
-
def array_to_datetime_with_tz(
ndarray values, tzinfo tz, bint dayfirst, bint yearfirst, NPY_DATETIMEUNIT creso
):
diff --git a/pandas/_libs/tslibs/dtypes.pxd b/pandas/_libs/tslibs/dtypes.pxd
index 88cfa6ca60d93..33f6789f3b402 100644
--- a/pandas/_libs/tslibs/dtypes.pxd
+++ b/pandas/_libs/tslibs/dtypes.pxd
@@ -11,7 +11,6 @@ cpdef int64_t periods_per_second(NPY_DATETIMEUNIT reso) except? -1
cdef NPY_DATETIMEUNIT get_supported_reso(NPY_DATETIMEUNIT reso)
cdef bint is_supported_unit(NPY_DATETIMEUNIT reso)
-cpdef freq_to_period_freqstr(freq_n, freq_name)
cdef dict c_OFFSET_TO_PERIOD_FREQSTR
cdef dict c_OFFSET_DEPR_FREQSTR
cdef dict c_REVERSE_OFFSET_DEPR_FREQSTR
diff --git a/pandas/_libs/tslibs/dtypes.pyi b/pandas/_libs/tslibs/dtypes.pyi
index 7fdeb88d498ac..b435b9e2d8b31 100644
--- a/pandas/_libs/tslibs/dtypes.pyi
+++ b/pandas/_libs/tslibs/dtypes.pyi
@@ -1,18 +1,19 @@
from enum import Enum
+from pandas._typing import Self
+
OFFSET_TO_PERIOD_FREQSTR: dict[str, str]
def periods_per_day(reso: int = ...) -> int: ...
def periods_per_second(reso: int) -> int: ...
def abbrev_to_npy_unit(abbrev: str | None) -> int: ...
-def freq_to_period_freqstr(freq_n: int, freq_name: str) -> str: ...
class PeriodDtypeBase:
_dtype_code: int # PeriodDtypeCode
_n: int
# actually __cinit__
- def __new__(cls, code: int, n: int): ...
+ def __new__(cls, code: int, n: int) -> Self: ...
@property
def _freq_group_code(self) -> int: ...
@property
diff --git a/pandas/_libs/tslibs/dtypes.pyx b/pandas/_libs/tslibs/dtypes.pyx
index 52e1133e596c5..5bfbe211bfd14 100644
--- a/pandas/_libs/tslibs/dtypes.pyx
+++ b/pandas/_libs/tslibs/dtypes.pyx
@@ -273,19 +273,6 @@ cdef dict c_OFFSET_DEPR_FREQSTR = {
"Y-SEP": "YE-SEP",
"Y-OCT": "YE-OCT",
"Y-NOV": "YE-NOV",
- "A": "YE",
- "A-DEC": "YE-DEC",
- "A-JAN": "YE-JAN",
- "A-FEB": "YE-FEB",
- "A-MAR": "YE-MAR",
- "A-APR": "YE-APR",
- "A-MAY": "YE-MAY",
- "A-JUN": "YE-JUN",
- "A-JUL": "YE-JUL",
- "A-AUG": "YE-AUG",
- "A-SEP": "YE-SEP",
- "A-OCT": "YE-OCT",
- "A-NOV": "YE-NOV",
"BY": "BYE",
"BY-DEC": "BYE-DEC",
"BY-JAN": "BYE-JAN",
@@ -299,19 +286,6 @@ cdef dict c_OFFSET_DEPR_FREQSTR = {
"BY-SEP": "BYE-SEP",
"BY-OCT": "BYE-OCT",
"BY-NOV": "BYE-NOV",
- "BA": "BYE",
- "BA-DEC": "BYE-DEC",
- "BA-JAN": "BYE-JAN",
- "BA-FEB": "BYE-FEB",
- "BA-MAR": "BYE-MAR",
- "BA-APR": "BYE-APR",
- "BA-MAY": "BYE-MAY",
- "BA-JUN": "BYE-JUN",
- "BA-JUL": "BYE-JUL",
- "BA-AUG": "BYE-AUG",
- "BA-SEP": "BYE-SEP",
- "BA-OCT": "BYE-OCT",
- "BA-NOV": "BYE-NOV",
"BM": "BME",
"CBM": "CBME",
"SM": "SME",
@@ -334,82 +308,12 @@ cdef dict c_REVERSE_OFFSET_DEPR_FREQSTR = {
v: k for k, v in c_OFFSET_DEPR_FREQSTR.items()
}
-cpdef freq_to_period_freqstr(freq_n, freq_name):
- if freq_n == 1:
- freqstr = f"""{c_OFFSET_TO_PERIOD_FREQSTR.get(
- freq_name, freq_name)}"""
- else:
- freqstr = f"""{freq_n}{c_OFFSET_TO_PERIOD_FREQSTR.get(
- freq_name, freq_name)}"""
- return freqstr
-
# Map deprecated resolution abbreviations to correct resolution abbreviations
cdef dict c_DEPR_ABBREVS = {
- "A": "Y",
- "a": "Y",
- "A-DEC": "Y-DEC",
- "A-JAN": "Y-JAN",
- "A-FEB": "Y-FEB",
- "A-MAR": "Y-MAR",
- "A-APR": "Y-APR",
- "A-MAY": "Y-MAY",
- "A-JUN": "Y-JUN",
- "A-JUL": "Y-JUL",
- "A-AUG": "Y-AUG",
- "A-SEP": "Y-SEP",
- "A-OCT": "Y-OCT",
- "A-NOV": "Y-NOV",
- "BA": "BY",
- "BA-DEC": "BY-DEC",
- "BA-JAN": "BY-JAN",
- "BA-FEB": "BY-FEB",
- "BA-MAR": "BY-MAR",
- "BA-APR": "BY-APR",
- "BA-MAY": "BY-MAY",
- "BA-JUN": "BY-JUN",
- "BA-JUL": "BY-JUL",
- "BA-AUG": "BY-AUG",
- "BA-SEP": "BY-SEP",
- "BA-OCT": "BY-OCT",
- "BA-NOV": "BY-NOV",
- "AS": "YS",
- "AS-DEC": "YS-DEC",
- "AS-JAN": "YS-JAN",
- "AS-FEB": "YS-FEB",
- "AS-MAR": "YS-MAR",
- "AS-APR": "YS-APR",
- "AS-MAY": "YS-MAY",
- "AS-JUN": "YS-JUN",
- "AS-JUL": "YS-JUL",
- "AS-AUG": "YS-AUG",
- "AS-SEP": "YS-SEP",
- "AS-OCT": "YS-OCT",
- "AS-NOV": "YS-NOV",
- "BAS": "BYS",
- "BAS-DEC": "BYS-DEC",
- "BAS-JAN": "BYS-JAN",
- "BAS-FEB": "BYS-FEB",
- "BAS-MAR": "BYS-MAR",
- "BAS-APR": "BYS-APR",
- "BAS-MAY": "BYS-MAY",
- "BAS-JUN": "BYS-JUN",
- "BAS-JUL": "BYS-JUL",
- "BAS-AUG": "BYS-AUG",
- "BAS-SEP": "BYS-SEP",
- "BAS-OCT": "BYS-OCT",
- "BAS-NOV": "BYS-NOV",
"H": "h",
"BH": "bh",
"CBH": "cbh",
- "T": "min",
- "t": "min",
"S": "s",
- "L": "ms",
- "l": "ms",
- "U": "us",
- "u": "us",
- "N": "ns",
- "n": "ns",
}
@@ -503,13 +407,17 @@ class Resolution(Enum):
"""
cdef:
str abbrev
+ if freq in {"T", "t", "L", "l", "U", "u", "N", "n"}:
+ raise ValueError(
+ f"Frequency \'{freq}\' is no longer supported."
+ )
try:
if freq in c_DEPR_ABBREVS:
abbrev = c_DEPR_ABBREVS[freq]
warnings.warn(
f"\'{freq}\' is deprecated and will be removed in a future "
f"version. Please use \'{abbrev}\' "
- "instead of \'{freq}\'.",
+ f"instead of \'{freq}\'.",
FutureWarning,
stacklevel=find_stack_level(),
)
diff --git a/pandas/_libs/tslibs/nattype.pyi b/pandas/_libs/tslibs/nattype.pyi
index bd86a6fdc2174..f49e894a0bfec 100644
--- a/pandas/_libs/tslibs/nattype.pyi
+++ b/pandas/_libs/tslibs/nattype.pyi
@@ -1,20 +1,30 @@
from datetime import (
+ date as date_,
datetime,
+ time as time_,
timedelta,
tzinfo as _tzinfo,
)
-import typing
+from typing import (
+ Literal,
+ NoReturn,
+ TypeAlias,
+)
import numpy as np
from pandas._libs.tslibs.period import Period
-from pandas._typing import Self
+from pandas._typing import (
+ Frequency,
+ Self,
+ TimestampNonexistent,
+)
NaT: NaTType
iNaT: int
nat_strings: set[str]
-_NaTComparisonTypes: typing.TypeAlias = (
+_NaTComparisonTypes: TypeAlias = (
datetime | timedelta | Period | np.datetime64 | np.timedelta64
)
@@ -61,18 +71,38 @@ class NaTType:
def week(self) -> float: ...
@property
def weekofyear(self) -> float: ...
+ @property
+ def fold(self) -> int: ...
def day_name(self) -> float: ...
def month_name(self) -> float: ...
def weekday(self) -> float: ...
def isoweekday(self) -> float: ...
+ def isoformat(self, sep: str = ..., timespec: str = ...) -> str: ...
+ def strftime(self, format: str) -> NoReturn: ...
def total_seconds(self) -> float: ...
def today(self, *args, **kwargs) -> NaTType: ...
def now(self, *args, **kwargs) -> NaTType: ...
def to_pydatetime(self) -> NaTType: ...
def date(self) -> NaTType: ...
- def round(self) -> NaTType: ...
- def floor(self) -> NaTType: ...
- def ceil(self) -> NaTType: ...
+ def round(
+ self,
+ freq: Frequency,
+ ambiguous: bool | Literal["raise"] | NaTType = ...,
+ nonexistent: TimestampNonexistent = ...,
+ ) -> NaTType: ...
+ def floor(
+ self,
+ freq: Frequency,
+ ambiguous: bool | Literal["raise"] | NaTType = ...,
+ nonexistent: TimestampNonexistent = ...,
+ ) -> NaTType: ...
+ def ceil(
+ self,
+ freq: Frequency,
+ ambiguous: bool | Literal["raise"] | NaTType = ...,
+ nonexistent: TimestampNonexistent = ...,
+ ) -> NaTType: ...
+ def combine(cls, date: date_, time: time_) -> NoReturn: ...
@property
def tzinfo(self) -> None: ...
@property
@@ -81,8 +111,8 @@ class NaTType:
def tz_localize(
self,
tz: _tzinfo | str | None,
- ambiguous: str = ...,
- nonexistent: str = ...,
+ ambiguous: bool | Literal["raise"] | NaTType = ...,
+ nonexistent: TimestampNonexistent = ...,
) -> NaTType: ...
def replace(
self,
@@ -121,6 +151,8 @@ class NaTType:
@property
def days(self) -> float: ...
@property
+ def seconds(self) -> float: ...
+ @property
def microseconds(self) -> float: ...
@property
def nanoseconds(self) -> float: ...
diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx
index 65db0e05f859c..7a08e4ad4b260 100644
--- a/pandas/_libs/tslibs/nattype.pyx
+++ b/pandas/_libs/tslibs/nattype.pyx
@@ -76,7 +76,7 @@ cdef _nat_rdivide_op(self, other):
return NotImplemented
-def __nat_unpickle(*args):
+def _nat_unpickle(*args):
# return constant defined in the module
return c_NaT
@@ -360,7 +360,7 @@ class NaTType(_NaT):
return self.__reduce__()
def __reduce__(self):
- return (__nat_unpickle, (None, ))
+ return (_nat_unpickle, (None, ))
def __rtruediv__(self, other):
return _nat_rdivide_op(self, other)
@@ -973,16 +973,16 @@ timedelta}, default 'raise'
A timestamp can be rounded using multiple frequency units:
- >>> ts.round(freq='h') # hour
+ >>> ts.round(freq='h') # hour
Timestamp('2020-03-14 16:00:00')
- >>> ts.round(freq='min') # minute
+ >>> ts.round(freq='min') # minute
Timestamp('2020-03-14 15:33:00')
- >>> ts.round(freq='s') # seconds
+ >>> ts.round(freq='s') # seconds
Timestamp('2020-03-14 15:32:52')
- >>> ts.round(freq='ms') # milliseconds
+ >>> ts.round(freq='ms') # milliseconds
Timestamp('2020-03-14 15:32:52.193000')
``freq`` can also be a multiple of a single unit, like '5min' (i.e. 5 minutes):
@@ -1062,16 +1062,16 @@ timedelta}, default 'raise'
A timestamp can be floored using multiple frequency units:
- >>> ts.floor(freq='h') # hour
+ >>> ts.floor(freq='h') # hour
Timestamp('2020-03-14 15:00:00')
- >>> ts.floor(freq='min') # minute
+ >>> ts.floor(freq='min') # minute
Timestamp('2020-03-14 15:32:00')
- >>> ts.floor(freq='s') # seconds
+ >>> ts.floor(freq='s') # seconds
Timestamp('2020-03-14 15:32:52')
- >>> ts.floor(freq='ns') # nanoseconds
+ >>> ts.floor(freq='ns') # nanoseconds
Timestamp('2020-03-14 15:32:52.192548651')
``freq`` can also be a multiple of a single unit, like '5min' (i.e. 5 minutes):
@@ -1151,16 +1151,16 @@ timedelta}, default 'raise'
A timestamp can be ceiled using multiple frequency units:
- >>> ts.ceil(freq='h') # hour
+ >>> ts.ceil(freq='h') # hour
Timestamp('2020-03-14 16:00:00')
- >>> ts.ceil(freq='min') # minute
+ >>> ts.ceil(freq='min') # minute
Timestamp('2020-03-14 15:33:00')
- >>> ts.ceil(freq='s') # seconds
+ >>> ts.ceil(freq='s') # seconds
Timestamp('2020-03-14 15:32:53')
- >>> ts.ceil(freq='us') # microseconds
+ >>> ts.ceil(freq='us') # microseconds
Timestamp('2020-03-14 15:32:52.192549')
``freq`` can also be a multiple of a single unit, like '5min' (i.e. 5 minutes):
diff --git a/pandas/_libs/tslibs/offsets.pyi b/pandas/_libs/tslibs/offsets.pyi
index 7eb8dc0813868..3f942d6aa3622 100644
--- a/pandas/_libs/tslibs/offsets.pyi
+++ b/pandas/_libs/tslibs/offsets.pyi
@@ -20,8 +20,6 @@ from pandas._typing import (
npt,
)
-from .timedeltas import Timedelta
-
_BaseOffsetT = TypeVar("_BaseOffsetT", bound=BaseOffset)
_DatetimeT = TypeVar("_DatetimeT", bound=datetime)
_TimedeltaT = TypeVar("_TimedeltaT", bound=timedelta)
@@ -64,7 +62,7 @@ class BaseOffset:
@overload
def __rsub__(self, other: npt.NDArray[np.object_]) -> npt.NDArray[np.object_]: ...
@overload
- def __rsub__(self, other: BaseOffset): ...
+ def __rsub__(self, other: BaseOffset) -> Self: ...
@overload
def __rsub__(self, other: _DatetimeT) -> _DatetimeT: ...
@overload
@@ -72,7 +70,7 @@ class BaseOffset:
@overload
def __mul__(self, other: np.ndarray) -> np.ndarray: ...
@overload
- def __mul__(self, other: int): ...
+ def __mul__(self, other: int) -> Self: ...
@overload
def __rmul__(self, other: np.ndarray) -> np.ndarray: ...
@overload
@@ -94,13 +92,12 @@ class BaseOffset:
def __getstate__(self): ...
@property
def nanos(self) -> int: ...
- def is_anchored(self) -> bool: ...
def _get_offset(name: str) -> BaseOffset: ...
class SingleConstructorOffset(BaseOffset):
@classmethod
- def _from_name(cls, suffix: None = ...): ...
+ def _from_name(cls, suffix: None = ...) -> Self: ...
def __reduce__(self): ...
@overload
@@ -115,8 +112,6 @@ class Tick(SingleConstructorOffset):
_prefix: str
def __init__(self, n: int = ..., normalize: bool = ...) -> None: ...
@property
- def delta(self) -> Timedelta: ...
- @property
def nanos(self) -> int: ...
def delta_to_tick(delta: timedelta) -> Tick: ...
@@ -197,7 +192,10 @@ class WeekOfMonth(WeekOfMonthMixin):
self, n: int = ..., normalize: bool = ..., week: int = ..., weekday: int = ...
) -> None: ...
-class LastWeekOfMonth(WeekOfMonthMixin): ...
+class LastWeekOfMonth(WeekOfMonthMixin):
+ def __init__(
+ self, n: int = ..., normalize: bool = ..., weekday: int = ...
+ ) -> None: ...
class FY5253Mixin(SingleConstructorOffset):
def __init__(
diff --git a/pandas/_libs/tslibs/offsets.pyx b/pandas/_libs/tslibs/offsets.pyx
index b3788b6003e67..e36abdf0ad971 100644
--- a/pandas/_libs/tslibs/offsets.pyx
+++ b/pandas/_libs/tslibs/offsets.pyx
@@ -755,27 +755,21 @@ cdef class BaseOffset:
def nanos(self):
raise ValueError(f"{self} is a non-fixed frequency")
- def is_anchored(self) -> bool:
- # TODO: Does this make sense for the general case? It would help
- # if there were a canonical docstring for what is_anchored means.
- """
- Return boolean whether the frequency is a unit frequency (n=1).
-
- Examples
- --------
- >>> pd.DateOffset().is_anchored()
- True
- >>> pd.DateOffset(2).is_anchored()
- False
- """
- return self.n == 1
-
# ------------------------------------------------------------------
def is_month_start(self, _Timestamp ts):
"""
Return boolean whether a timestamp occurs on the month start.
+ Parameters
+ ----------
+ ts : Timestamp
+ The timestamp to check.
+
+ See Also
+ --------
+ is_month_end : Return boolean whether a timestamp occurs on the month end.
+
Examples
--------
>>> ts = pd.Timestamp(2022, 1, 1)
@@ -789,6 +783,15 @@ cdef class BaseOffset:
"""
Return boolean whether a timestamp occurs on the month end.
+ Parameters
+ ----------
+ ts : Timestamp
+ The timestamp to check.
+
+ See Also
+ --------
+ is_month_start : Return boolean whether a timestamp occurs on the month start.
+
Examples
--------
>>> ts = pd.Timestamp(2022, 1, 1)
@@ -802,6 +805,15 @@ cdef class BaseOffset:
"""
Return boolean whether a timestamp occurs on the quarter start.
+ Parameters
+ ----------
+ ts : Timestamp
+ The timestamp to check.
+
+ See Also
+ --------
+ is_quarter_end : Return boolean whether a timestamp occurs on the quarter end.
+
Examples
--------
>>> ts = pd.Timestamp(2022, 1, 1)
@@ -815,6 +827,16 @@ cdef class BaseOffset:
"""
Return boolean whether a timestamp occurs on the quarter end.
+ Parameters
+ ----------
+ ts : Timestamp
+ The timestamp to check.
+
+ See Also
+ --------
+ is_quarter_start : Return boolean whether a timestamp
+ occurs on the quarter start.
+
Examples
--------
>>> ts = pd.Timestamp(2022, 1, 1)
@@ -828,6 +850,15 @@ cdef class BaseOffset:
"""
Return boolean whether a timestamp occurs on the year start.
+ Parameters
+ ----------
+ ts : Timestamp
+ The timestamp to check.
+
+ See Also
+ --------
+ is_year_end : Return boolean whether a timestamp occurs on the year end.
+
Examples
--------
>>> ts = pd.Timestamp(2022, 1, 1)
@@ -841,6 +872,15 @@ cdef class BaseOffset:
"""
Return boolean whether a timestamp occurs on the year end.
+ Parameters
+ ----------
+ ts : Timestamp
+ The timestamp to check.
+
+ See Also
+ --------
+ is_year_start : Return boolean whether a timestamp occurs on the year start.
+
Examples
--------
>>> ts = pd.Timestamp(2022, 1, 1)
@@ -917,22 +957,6 @@ cdef class Tick(SingleConstructorOffset):
def _as_pd_timedelta(self):
return Timedelta(self)
- @property
- def delta(self):
- warnings.warn(
- # GH#55498
- f"{type(self).__name__}.delta is deprecated and will be removed in "
- "a future version. Use pd.Timedelta(obj) instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- try:
- return self.n * Timedelta(self._nanos_inc)
- except OverflowError as err:
- # GH#55503 as_unit will raise a more useful OutOfBoundsTimedelta
- Timedelta(self).as_unit("ns")
- raise AssertionError("This should not be reached.")
-
@property
def nanos(self) -> int64_t:
"""
@@ -953,9 +977,6 @@ cdef class Tick(SingleConstructorOffset):
def is_on_offset(self, dt: datetime) -> bool:
return True
- def is_anchored(self) -> bool:
- return False
-
# This is identical to BaseOffset.__hash__, but has to be redefined here
# for Python 3, because we've redefined __eq__.
def __hash__(self) -> int:
@@ -1428,13 +1449,22 @@ cdef class RelativeDeltaOffset(BaseOffset):
"minutes",
"seconds",
"microseconds",
+ "milliseconds",
}
# relativedelta/_offset path only valid for base DateOffset
if self._use_relativedelta and set(kwds).issubset(relativedelta_fast):
+ td_args = {
+ "days",
+ "hours",
+ "minutes",
+ "seconds",
+ "microseconds",
+ "milliseconds"
+ }
td_kwds = {
key: val
for key, val in kwds.items()
- if key in ["days", "hours", "minutes", "seconds", "microseconds"]
+ if key in td_args
}
if "weeks" in kwds:
days = td_kwds.get("days", 0)
@@ -1444,6 +1474,8 @@ cdef class RelativeDeltaOffset(BaseOffset):
delta = Timedelta(**td_kwds)
if "microseconds" in kwds:
delta = delta.as_unit("us")
+ elif "milliseconds" in kwds:
+ delta = delta.as_unit("ms")
else:
delta = delta.as_unit("s")
else:
@@ -1461,6 +1493,8 @@ cdef class RelativeDeltaOffset(BaseOffset):
delta = Timedelta(self._offset * self.n)
if "microseconds" in kwds:
delta = delta.as_unit("us")
+ elif "milliseconds" in kwds:
+ delta = delta.as_unit("ms")
else:
delta = delta.as_unit("s")
return delta
@@ -1514,7 +1548,7 @@ class DateOffset(RelativeDeltaOffset, metaclass=OffsetMeta):
Standard kind of date increment used for a date range.
Works exactly like the keyword argument form of relativedelta.
- Note that the positional argument form of relativedelata is not
+ Note that the positional argument form of relativedelta is not
supported. Use of the keyword n is discouraged-- you would be better
off specifying n in the keywords you use, but regardless it is
there for you. n is needed for DateOffset subclasses.
@@ -2662,9 +2696,6 @@ cdef class QuarterOffset(SingleConstructorOffset):
month = MONTH_ALIASES[self.startingMonth]
return f"{self._prefix}-{month}"
- def is_anchored(self) -> bool:
- return self.n == 1 and self.startingMonth is not None
-
def is_on_offset(self, dt: datetime) -> bool:
if self.normalize and not _is_normalized(dt):
return False
@@ -3307,9 +3338,6 @@ cdef class Week(SingleConstructorOffset):
self.weekday = state.pop("weekday")
self._cache = state.pop("_cache", {})
- def is_anchored(self) -> bool:
- return self.n == 1 and self.weekday is not None
-
@apply_wraps
def _apply(self, other):
if self.weekday is None:
@@ -3596,11 +3624,6 @@ cdef class FY5253Mixin(SingleConstructorOffset):
self.weekday = state.pop("weekday")
self.variation = state.pop("variation")
- def is_anchored(self) -> bool:
- return (
- self.n == 1 and self.startingMonth is not None and self.weekday is not None
- )
-
# --------------------------------------------------------------------
# Name-related methods
@@ -4221,9 +4244,7 @@ cdef class CustomBusinessDay(BusinessDay):
@property
def _period_dtype_code(self):
# GH#52534
- raise TypeError(
- "CustomBusinessDay is not supported as period frequency"
- )
+ raise ValueError(f"{self.base} is not supported as period frequency")
_apply_array = BaseOffset._apply_array
@@ -4661,29 +4682,7 @@ _lite_rule_alias = {
"ns": "ns",
}
-_dont_uppercase = {
- "h",
- "bh",
- "cbh",
- "MS",
- "ms",
- "s",
- "me",
- "qe",
- "qe-dec",
- "qe-jan",
- "qe-feb",
- "qe-mar",
- "qe-apr",
- "qe-may",
- "qe-jun",
- "qe-jul",
- "qe-aug",
- "qe-sep",
- "qe-oct",
- "qe-nov",
- "ye",
-}
+_dont_uppercase = {"h", "bh", "cbh", "MS", "ms", "s"}
INVALID_FREQ_ERR_MSG = "Invalid frequency: {0}"
@@ -4702,7 +4701,29 @@ def _get_offset(name: str) -> BaseOffset:
--------
_get_offset('EOM') --> BMonthEnd(1)
"""
- if name.lower() not in _dont_uppercase:
+ if (
+ name not in _lite_rule_alias
+ and (name.upper() in _lite_rule_alias)
+ and name != "ms"
+ ):
+ warnings.warn(
+ f"\'{name}\' is deprecated and will be removed "
+ f"in a future version, please use \'{name.upper()}\' instead.",
+ FutureWarning,
+ stacklevel=find_stack_level(),
+ )
+ elif (
+ name not in _lite_rule_alias
+ and (name.lower() in _lite_rule_alias)
+ and name != "MS"
+ ):
+ warnings.warn(
+ f"\'{name}\' is deprecated and will be removed "
+ f"in a future version, please use \'{name.lower()}\' instead.",
+ FutureWarning,
+ stacklevel=find_stack_level(),
+ )
+ if name not in _dont_uppercase:
name = name.upper()
name = _lite_rule_alias.get(name, name)
name = _lite_rule_alias.get(name.lower(), name)
@@ -4734,6 +4755,10 @@ cpdef to_offset(freq, bint is_period=False):
Parameters
----------
freq : str, datetime.timedelta, BaseOffset or None
+ The frequency represented.
+ is_period : bool, default False
+ Convert string denoting period frequency to corresponding offsets
+ frequency if is_period=True.
Returns
-------
@@ -4768,23 +4793,34 @@ cpdef to_offset(freq, bint is_period=False):
>>> to_offset(pd.offsets.Hour())
+
+ Passing the parameter ``is_period`` equal to True, you can use a string
+ denoting period frequency:
+
+ >>> freq = to_offset(freq="ME", is_period=False)
+ >>> freq.rule_code
+ 'ME'
+
+ >>> freq = to_offset(freq="M", is_period=True)
+ >>> freq.rule_code
+ 'ME'
"""
if freq is None:
return None
- if isinstance(freq, BaseOffset):
- return freq
-
if isinstance(freq, tuple):
raise TypeError(
f"to_offset does not support tuples {freq}, pass as a string instead"
)
+ if isinstance(freq, BaseOffset):
+ result = freq
+
elif PyDelta_Check(freq):
- return delta_to_tick(freq)
+ result = delta_to_tick(freq)
elif isinstance(freq, str):
- delta = None
+ result = None
stride_sign = None
try:
@@ -4795,40 +4831,53 @@ cpdef to_offset(freq, bint is_period=False):
tups = zip(split[0::4], split[1::4], split[2::4])
for n, (sep, stride, name) in enumerate(tups):
- if is_period is False and name in c_OFFSET_DEPR_FREQSTR:
+ if not is_period and name.upper() in c_OFFSET_DEPR_FREQSTR:
+ warnings.warn(
+ f"\'{name}\' is deprecated and will be removed "
+ f"in a future version, please use "
+ f"\'{c_OFFSET_DEPR_FREQSTR.get(name.upper())}\' instead.",
+ FutureWarning,
+ stacklevel=find_stack_level(),
+ )
+ name = c_OFFSET_DEPR_FREQSTR[name.upper()]
+ if (not is_period and
+ name != name.upper() and
+ name.lower() not in {"s", "ms", "us", "ns"} and
+ name.upper().split("-")[0].endswith(("S", "E"))):
warnings.warn(
f"\'{name}\' is deprecated and will be removed "
f"in a future version, please use "
- f"\'{c_OFFSET_DEPR_FREQSTR.get(name)}\' instead.",
+ f"\'{name.upper()}\' instead.",
FutureWarning,
stacklevel=find_stack_level(),
)
- name = c_OFFSET_DEPR_FREQSTR[name]
- if is_period is True and name in c_REVERSE_OFFSET_DEPR_FREQSTR:
- if name.startswith("Y"):
+ name = name.upper()
+ if is_period and name.upper() in c_REVERSE_OFFSET_DEPR_FREQSTR:
+ if name.upper().startswith("Y"):
raise ValueError(
- f"for Period, please use \'Y{name[2:]}\' "
+ f"for Period, please use \'Y{name.upper()[2:]}\' "
f"instead of \'{name}\'"
)
- if (name.startswith("B") or
- name.startswith("S") or name.startswith("C")):
+ if (name.upper().startswith("B") or
+ name.upper().startswith("S") or
+ name.upper().startswith("C")):
raise ValueError(INVALID_FREQ_ERR_MSG.format(name))
else:
raise ValueError(
f"for Period, please use "
- f"\'{c_REVERSE_OFFSET_DEPR_FREQSTR.get(name)}\' "
+ f"\'{c_REVERSE_OFFSET_DEPR_FREQSTR.get(name.upper())}\' "
f"instead of \'{name}\'"
)
- elif is_period is True and name in c_OFFSET_DEPR_FREQSTR:
- if name.startswith("A"):
+ elif is_period and name.upper() in c_OFFSET_DEPR_FREQSTR:
+ if name.upper() != name:
warnings.warn(
- f"\'{name}\' is deprecated and will be removed in a future "
- f"version, please use \'{c_DEPR_ABBREVS.get(name)}\' "
+ f"\'{name}\' is deprecated and will be removed in "
+ f"a future version, please use \'{name.upper()}\' "
f"instead.",
FutureWarning,
stacklevel=find_stack_level(),
)
- name = c_OFFSET_DEPR_FREQSTR.get(name)
+ name = c_OFFSET_DEPR_FREQSTR.get(name.upper())
if sep != "" and not sep.isspace():
raise ValueError("separator must be spaces")
@@ -4864,21 +4913,27 @@ cpdef to_offset(freq, bint is_period=False):
offset = _get_offset(prefix)
offset = offset * int(np.fabs(stride) * stride_sign)
- if delta is None:
- delta = offset
+ if result is None:
+ result = offset
else:
- delta = delta + offset
+ result = result + offset
except (ValueError, TypeError) as err:
raise ValueError(INVALID_FREQ_ERR_MSG.format(
f"{freq}, failed to parse with error message: {repr(err)}")
)
else:
- delta = None
+ result = None
- if delta is None:
+ if result is None:
raise ValueError(INVALID_FREQ_ERR_MSG.format(freq))
- return delta
+ if is_period and not hasattr(result, "_period_dtype_code"):
+ if isinstance(freq, str):
+ raise ValueError(f"{result.name} is not supported as period frequency")
+ else:
+ raise ValueError(f"{freq} is not supported as period frequency")
+
+ return result
# ----------------------------------------------------------------------
diff --git a/pandas/_libs/tslibs/parsing.pyx b/pandas/_libs/tslibs/parsing.pyx
index d0872a509c440..384df1cac95eb 100644
--- a/pandas/_libs/tslibs/parsing.pyx
+++ b/pandas/_libs/tslibs/parsing.pyx
@@ -45,7 +45,6 @@ from decimal import InvalidOperation
from dateutil.parser import DEFAULTPARSER
from dateutil.tz import (
- tzlocal as _dateutil_tzlocal,
tzoffset,
tzutc as _dateutil_tzutc,
)
@@ -703,17 +702,12 @@ cdef datetime dateutil_parse(
if res.tzname and res.tzname in time.tzname:
# GH#50791
if res.tzname != "UTC":
- # If the system is localized in UTC (as many CI runs are)
- # we get tzlocal, once the deprecation is enforced will get
- # timezone.utc, not raise.
- warnings.warn(
+ raise ValueError(
f"Parsing '{res.tzname}' as tzlocal (dependent on system timezone) "
- "is deprecated and will raise in a future version. Pass the 'tz' "
+ "is no longer supported. Pass the 'tz' "
"keyword or call tz_localize after construction instead",
- FutureWarning,
- stacklevel=find_stack_level()
)
- ret = ret.replace(tzinfo=_dateutil_tzlocal())
+ ret = ret.replace(tzinfo=timezone.utc)
elif res.tzoffset == 0:
ret = ret.replace(tzinfo=_dateutil_tzutc())
elif res.tzoffset:
@@ -733,15 +727,10 @@ cdef datetime dateutil_parse(
)
elif res.tzname is not None:
# e.g. "1994 Jan 15 05:16 FOO" where FOO is not recognized
- # GH#18702
- warnings.warn(
+ # GH#18702, # GH 50235 enforced in 3.0
+ raise ValueError(
f'Parsed string "{timestr}" included an un-recognized timezone '
- f'"{res.tzname}". Dropping unrecognized timezones is deprecated; '
- "in a future version this will raise. Instead pass the string "
- "without the timezone, then use .tz_localize to convert to a "
- "recognized timezone.",
- FutureWarning,
- stacklevel=find_stack_level()
+ f'"{res.tzname}".'
)
out_bestunit[0] = attrname_to_npy_unit[reso]
@@ -920,8 +909,8 @@ def guess_datetime_format(dt_str: str, bint dayfirst=False) -> str | None:
(("year", "month", "day", "hour"), "%Y%m%d%H", 0),
(("year", "month", "day"), "%Y%m%d", 0),
(("hour", "minute", "second"), "%H%M%S", 0),
- (("hour", "minute"), "%H%M", 0),
(("year",), "%Y", 0),
+ (("hour", "minute"), "%H%M", 0),
(("month",), "%B", 0),
(("month",), "%b", 0),
(("month",), "%m", 2),
@@ -941,8 +930,10 @@ def guess_datetime_format(dt_str: str, bint dayfirst=False) -> str | None:
datetime_attrs_to_format.remove(day_attribute_and_format)
datetime_attrs_to_format.insert(0, day_attribute_and_format)
- # same default used by dateutil
- default = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
+ # Use this instead of the dateutil default of
+ # `datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)`
+ # as that causes issues on the 29th of February.
+ default = datetime(1970, 1, 1)
try:
parsed_datetime = dateutil_parse(
dt_str,
diff --git a/pandas/_libs/tslibs/period.pyx b/pandas/_libs/tslibs/period.pyx
index 74804336f09a3..838b5b9f4595f 100644
--- a/pandas/_libs/tslibs/period.pyx
+++ b/pandas/_libs/tslibs/period.pyx
@@ -38,10 +38,7 @@ from libc.time cimport (
tm,
)
-from pandas._libs.tslibs.dtypes cimport (
- c_OFFSET_TO_PERIOD_FREQSTR,
- freq_to_period_freqstr,
-)
+from pandas._libs.tslibs.dtypes cimport c_OFFSET_TO_PERIOD_FREQSTR
from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime
@@ -97,9 +94,6 @@ from pandas._libs.tslibs.dtypes cimport (
attrname_to_abbrevs,
freq_group_code_to_npy_unit,
)
-
-from pandas._libs.tslibs.dtypes import freq_to_period_freqstr
-
from pandas._libs.tslibs.parsing cimport quarter_to_myear
from pandas._libs.tslibs.parsing import parse_datetime_string_with_reso
@@ -685,6 +679,8 @@ cdef char* c_strftime(npy_datetimestruct *dts, char *fmt):
c_date.tm_isdst = -1
result = malloc(result_len * sizeof(char))
+ if result is NULL:
+ raise MemoryError()
strftime(result, result_len, fmt, &c_date)
@@ -1554,7 +1550,7 @@ def extract_ordinals(ndarray values, freq) -> np.ndarray:
# if we don't raise here, we'll segfault later!
raise TypeError("extract_ordinals values must be object-dtype")
- freqstr = freq_to_period_freqstr(freq.n, freq.name)
+ freqstr = PeriodDtypeBase(freq._period_dtype_code, freq.n)._freqstr
for i in range(n):
# Analogous to: p = values[i]
@@ -1722,8 +1718,15 @@ cdef class PeriodMixin:
condition = self.freq != other
if condition:
- freqstr = freq_to_period_freqstr(self.freq.n, self.freq.name)
- other_freqstr = freq_to_period_freqstr(other.n, other.name)
+ freqstr = PeriodDtypeBase(
+ self.freq._period_dtype_code, self.freq.n
+ )._freqstr
+ if hasattr(other, "_period_dtype_code"):
+ other_freqstr = PeriodDtypeBase(
+ other._period_dtype_code, other.n
+ )._freqstr
+ else:
+ other_freqstr = other.freqstr
msg = DIFFERENT_FREQ.format(
cls=type(self).__name__,
own_freq=freqstr,
@@ -2479,7 +2482,7 @@ cdef class _Period(PeriodMixin):
>>> pd.Period('2020-01', 'D').freqstr
'D'
"""
- freqstr = freq_to_period_freqstr(self.freq.n, self.freq.name)
+ freqstr = PeriodDtypeBase(self.freq._period_dtype_code, self.freq.n)._freqstr
return freqstr
def __repr__(self) -> str:
diff --git a/pandas/_libs/tslibs/strptime.pyx b/pandas/_libs/tslibs/strptime.pyx
index ee72b1311051e..5c9f1c770ea7f 100644
--- a/pandas/_libs/tslibs/strptime.pyx
+++ b/pandas/_libs/tslibs/strptime.pyx
@@ -58,7 +58,6 @@ from pandas._libs.tslibs.dtypes cimport (
)
from pandas._libs.tslibs.nattype cimport (
NPY_NAT,
- c_NaT as NaT,
c_nat_strings as nat_strings,
)
from pandas._libs.tslibs.np_datetime cimport (
@@ -132,7 +131,7 @@ cdef bint parse_today_now(
if infer_reso:
creso = NPY_DATETIMEUNIT.NPY_FR_us
if utc:
- ts = <_Timestamp>Timestamp.utcnow()
+ ts = <_Timestamp>Timestamp.now(timezone.utc)
iresult[0] = ts._as_creso(creso)._value
else:
# GH#18705 make sure to_datetime("now") matches Timestamp("now")
@@ -310,7 +309,7 @@ def array_strptime(
values : ndarray of string-like objects
fmt : string-like regex
exact : matches must be exact if True, search if False
- errors : string specifying error handling, {'raise', 'ignore', 'coerce'}
+ errors : string specifying error handling, {'raise', 'coerce'}
creso : NPY_DATETIMEUNIT, default NPY_FR_ns
Set to NPY_FR_GENERIC to infer a resolution.
"""
@@ -322,7 +321,6 @@ def array_strptime(
object val
bint seen_datetime_offset = False
bint is_raise = errors=="raise"
- bint is_ignore = errors=="ignore"
bint is_coerce = errors=="coerce"
bint is_same_offsets
set out_tzoffset_vals = set()
@@ -334,7 +332,7 @@ def array_strptime(
bint infer_reso = creso == NPY_DATETIMEUNIT.NPY_FR_GENERIC
DatetimeParseState state = DatetimeParseState(creso)
- assert is_raise or is_ignore or is_coerce
+ assert is_raise or is_coerce
_validate_fmt(fmt)
format_regex, locale_time = _get_format_regex(fmt)
@@ -504,20 +502,20 @@ def array_strptime(
if seen_datetime_offset and not utc:
is_same_offsets = len(out_tzoffset_vals) == 1
if not is_same_offsets or (state.found_naive or state.found_other):
- result2 = _array_strptime_object_fallback(
- values, fmt=fmt, exact=exact, errors=errors, utc=utc
+ raise ValueError(
+ "Mixed timezones detected. Pass utc=True in to_datetime "
+ "or tz='UTC' in DatetimeIndex to convert to a common timezone."
)
- return result2, None
elif tz_out is not None:
# GH#55693
tz_offset = out_tzoffset_vals.pop()
tz_out2 = timezone(timedelta(seconds=tz_offset))
if not tz_compare(tz_out, tz_out2):
- # e.g. test_to_datetime_mixed_offsets_with_utc_false_deprecated
- result2 = _array_strptime_object_fallback(
- values, fmt=fmt, exact=exact, errors=errors, utc=utc
+ # e.g. test_to_datetime_mixed_offsets_with_utc_false_removed
+ raise ValueError(
+ "Mixed timezones detected. Pass utc=True in to_datetime "
+ "or tz='UTC' in DatetimeIndex to convert to a common timezone."
)
- return result2, None
# e.g. test_guess_datetime_format_with_parseable_formats
else:
# e.g. test_to_datetime_iso8601_with_timezone_valid
@@ -526,10 +524,10 @@ def array_strptime(
elif not utc:
if tz_out and (state.found_other or state.found_naive_str):
# found_other indicates a tz-naive int, float, dt64, or date
- result2 = _array_strptime_object_fallback(
- values, fmt=fmt, exact=exact, errors=errors, utc=utc
+ raise ValueError(
+ "Mixed timezones detected. Pass utc=True in to_datetime "
+ "or tz='UTC' in DatetimeIndex to convert to a common timezone."
)
- return result2, None
if infer_reso:
if state.creso_ever_changed:
@@ -791,157 +789,6 @@ cdef tzinfo _parse_with_format(
return tz
-def _array_strptime_object_fallback(
- ndarray[object] values,
- str fmt,
- bint exact=True,
- errors="raise",
- bint utc=False,
-):
-
- cdef:
- Py_ssize_t i, n = len(values)
- npy_datetimestruct dts
- int64_t iresult
- object val
- tzinfo tz
- bint is_raise = errors=="raise"
- bint is_ignore = errors=="ignore"
- bint is_coerce = errors=="coerce"
- bint iso_format = format_is_iso(fmt)
- NPY_DATETIMEUNIT creso, out_bestunit, item_reso
- int out_local = 0, out_tzoffset = 0
- bint string_to_dts_succeeded = 0
-
- assert is_raise or is_ignore or is_coerce
-
- item_reso = NPY_DATETIMEUNIT.NPY_FR_GENERIC
- format_regex, locale_time = _get_format_regex(fmt)
-
- result = np.empty(n, dtype=object)
-
- dts.us = dts.ps = dts.as = 0
-
- for i in range(n):
- val = values[i]
- try:
- if isinstance(val, str):
- if len(val) == 0 or val in nat_strings:
- result[i] = NaT
- continue
- elif checknull_with_nat_and_na(val):
- result[i] = NaT
- continue
- elif PyDateTime_Check(val):
- result[i] = Timestamp(val)
- continue
- elif PyDate_Check(val):
- result[i] = Timestamp(val)
- continue
- elif cnp.is_datetime64_object(val):
- result[i] = Timestamp(val)
- continue
- elif (
- (is_integer_object(val) or is_float_object(val))
- and (val != val or val == NPY_NAT)
- ):
- result[i] = NaT
- continue
- else:
- val = str(val)
-
- if fmt == "ISO8601":
- string_to_dts_succeeded = not string_to_dts(
- val, &dts, &out_bestunit, &out_local,
- &out_tzoffset, False, None, False
- )
- elif iso_format:
- string_to_dts_succeeded = not string_to_dts(
- val, &dts, &out_bestunit, &out_local,
- &out_tzoffset, False, fmt, exact
- )
- if string_to_dts_succeeded:
- # No error reported by string_to_dts, pick back up
- # where we left off
- creso = get_supported_reso(out_bestunit)
- try:
- value = npy_datetimestruct_to_datetime(creso, &dts)
- except OverflowError as err:
- raise OutOfBoundsDatetime(
- f"Out of bounds nanosecond timestamp: {val}"
- ) from err
- if out_local == 1:
- tz = timezone(timedelta(minutes=out_tzoffset))
- value = tz_localize_to_utc_single(
- value, tz, ambiguous="raise", nonexistent=None, creso=creso
- )
- else:
- tz = None
- ts = Timestamp._from_value_and_reso(value, creso, tz)
- result[i] = ts
- continue
-
- if parse_today_now(val, &iresult, utc, NPY_FR_ns):
- result[i] = Timestamp(val)
- continue
-
- # Some ISO formats can't be parsed by string_to_dts
- # For example, 6-digit YYYYMD. So, if there's an error, and a format
- # was specified, then try the string-matching code below. If the format
- # specified was 'ISO8601', then we need to error, because
- # only string_to_dts handles mixed ISO8601 formats.
- if not string_to_dts_succeeded and fmt == "ISO8601":
- raise ValueError(f"Time data {val} is not ISO8601 format")
-
- tz = _parse_with_format(
- val, fmt, exact, format_regex, locale_time, &dts, &item_reso
- )
- try:
- iresult = npy_datetimestruct_to_datetime(item_reso, &dts)
- except OverflowError as err:
- raise OutOfBoundsDatetime(
- f"Out of bounds nanosecond timestamp: {val}"
- ) from err
- if tz is not None:
- iresult = tz_localize_to_utc_single(
- iresult, tz, ambiguous="raise", nonexistent=None, creso=item_reso
- )
- ts = Timestamp._from_value_and_reso(iresult, item_reso, tz)
- result[i] = ts
-
- except (ValueError, OutOfBoundsDatetime) as ex:
- ex.args = (
- f"{str(ex)}, at position {i}. You might want to try:\n"
- " - passing `format` if your strings have a consistent format;\n"
- " - passing `format='ISO8601'` if your strings are "
- "all ISO8601 but not necessarily in exactly the same format;\n"
- " - passing `format='mixed'`, and the format will be "
- "inferred for each element individually. "
- "You might want to use `dayfirst` alongside this.",
- )
- if is_coerce:
- result[i] = NaT
- continue
- elif is_raise:
- raise
- return values
-
- import warnings
-
- from pandas.util._exceptions import find_stack_level
- warnings.warn(
- "In a future version of pandas, parsing datetimes with mixed time "
- "zones will raise an error unless `utc=True`. Please specify `utc=True` "
- "to opt in to the new behaviour and silence this warning. "
- "To create a `Series` with mixed offsets and `object` dtype, "
- "please use `apply` and `datetime.datetime.strptime`",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- return result
-
-
class TimeRE(_TimeRE):
"""
Handle conversion from format directives to regexes.
diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx
index f6c69cf6d3875..9078fd4116899 100644
--- a/pandas/_libs/tslibs/timedeltas.pyx
+++ b/pandas/_libs/tslibs/timedeltas.pyx
@@ -1038,6 +1038,25 @@ cdef class _Timedelta(timedelta):
@property
def value(self):
+ """
+ Return the value of Timedelta object in nanoseconds.
+
+ Return the total seconds, milliseconds and microseconds
+ of the timedelta as nanoseconds.
+
+ Returns
+ -------
+ int
+
+ See Also
+ --------
+ Timedelta.unit : Return the unit of Timedelta object.
+
+ Examples
+ --------
+ >>> pd.Timedelta(1, "us").value
+ 1000
+ """
try:
return convert_reso(self._value, self._creso, NPY_FR_ns, False)
except OverflowError:
@@ -1120,6 +1139,37 @@ cdef class _Timedelta(timedelta):
def microseconds(self) -> int: # TODO(cython3): make cdef property
# NB: using the python C-API PyDateTime_DELTA_GET_MICROSECONDS will fail
# (or be incorrect)
+ """
+ Return the number of microseconds (n), where 0 <= n < 1 millisecond.
+
+ Timedelta.microseconds = milliseconds * 1000 + microseconds.
+
+ Returns
+ -------
+ int
+ Number of microseconds.
+
+ See Also
+ --------
+ Timedelta.components : Return all attributes with assigned values
+ (i.e. days, hours, minutes, seconds, milliseconds, microseconds,
+ nanoseconds).
+
+ Examples
+ --------
+ **Using string input**
+
+ >>> td = pd.Timedelta('1 days 2 min 3 us')
+
+ >>> td.microseconds
+ 3
+
+ **Using integer input**
+
+ >>> td = pd.Timedelta(42, unit='us')
+ >>> td.microseconds
+ 42
+ """
self._ensure_components()
return self._ms * 1000 + self._us
@@ -1141,6 +1191,26 @@ cdef class _Timedelta(timedelta):
@property
def unit(self) -> str:
+ """
+ Return the unit of Timedelta object.
+
+ The unit of Timedelta object is nanosecond, i.e., 'ns' by default.
+
+ Returns
+ -------
+ str
+
+ See Also
+ --------
+ Timedelta.value : Return the value of Timedelta object in nanoseconds.
+ Timedelta.as_unit : Convert the underlying int64 representation to
+ the given unit.
+
+ Examples
+ --------
+ >>> td = pd.Timedelta(42, unit='us')
+ 'ns'
+ """
return npy_unit_to_abbrev(self._creso)
def __hash__(_Timedelta self):
@@ -1306,7 +1376,10 @@ cdef class _Timedelta(timedelta):
datetime.timedelta(days=3)
"""
if self._creso == NPY_FR_ns:
- return timedelta(microseconds=int(self._value) / 1000)
+ us, remainder = divmod(self._value, 1000)
+ if remainder >= 500:
+ us += 1
+ return timedelta(microseconds=us)
# TODO(@WillAyd): is this the right way to use components?
self._ensure_components()
diff --git a/pandas/_libs/tslibs/timestamps.pyx b/pandas/_libs/tslibs/timestamps.pyx
index 568539b53aee0..d4cd90613ca5b 100644
--- a/pandas/_libs/tslibs/timestamps.pyx
+++ b/pandas/_libs/tslibs/timestamps.pyx
@@ -1418,6 +1418,14 @@ class Timestamp(_Timestamp):
>>> pd.Timestamp.utcnow() # doctest: +SKIP
Timestamp('2020-11-16 22:50:18.092888+0000', tz='UTC')
"""
+ warnings.warn(
+ # The stdlib datetime.utcnow is deprecated, so we deprecate to match.
+ # GH#56680
+ "Timestamp.utcnow is deprecated and will be removed in a future "
+ "version. Use Timestamp.now('UTC') instead.",
+ FutureWarning,
+ stacklevel=find_stack_level(),
+ )
return cls.now(UTC)
@classmethod
@@ -1438,6 +1446,14 @@ class Timestamp(_Timestamp):
Timestamp('2020-03-14 15:32:52+0000', tz='UTC')
"""
# GH#22451
+ warnings.warn(
+ # The stdlib datetime.utcfromtimestamp is deprecated, so we deprecate
+ # to match. GH#56680
+ "Timestamp.utcfromtimestamp is deprecated and will be removed in a "
+ "future version. Use Timestamp.fromtimestamp(ts, 'UTC') instead.",
+ FutureWarning,
+ stacklevel=find_stack_level(),
+ )
return cls.fromtimestamp(ts, tz="UTC")
@classmethod
@@ -1957,16 +1973,16 @@ timedelta}, default 'raise'
A timestamp can be rounded using multiple frequency units:
- >>> ts.round(freq='h') # hour
+ >>> ts.round(freq='h') # hour
Timestamp('2020-03-14 16:00:00')
- >>> ts.round(freq='min') # minute
+ >>> ts.round(freq='min') # minute
Timestamp('2020-03-14 15:33:00')
- >>> ts.round(freq='s') # seconds
+ >>> ts.round(freq='s') # seconds
Timestamp('2020-03-14 15:32:52')
- >>> ts.round(freq='ms') # milliseconds
+ >>> ts.round(freq='ms') # milliseconds
Timestamp('2020-03-14 15:32:52.193000')
``freq`` can also be a multiple of a single unit, like '5min' (i.e. 5 minutes):
@@ -2048,16 +2064,16 @@ timedelta}, default 'raise'
A timestamp can be floored using multiple frequency units:
- >>> ts.floor(freq='h') # hour
+ >>> ts.floor(freq='h') # hour
Timestamp('2020-03-14 15:00:00')
- >>> ts.floor(freq='min') # minute
+ >>> ts.floor(freq='min') # minute
Timestamp('2020-03-14 15:32:00')
- >>> ts.floor(freq='s') # seconds
+ >>> ts.floor(freq='s') # seconds
Timestamp('2020-03-14 15:32:52')
- >>> ts.floor(freq='ns') # nanoseconds
+ >>> ts.floor(freq='ns') # nanoseconds
Timestamp('2020-03-14 15:32:52.192548651')
``freq`` can also be a multiple of a single unit, like '5min' (i.e. 5 minutes):
@@ -2137,16 +2153,16 @@ timedelta}, default 'raise'
A timestamp can be ceiled using multiple frequency units:
- >>> ts.ceil(freq='h') # hour
+ >>> ts.ceil(freq='h') # hour
Timestamp('2020-03-14 16:00:00')
- >>> ts.ceil(freq='min') # minute
+ >>> ts.ceil(freq='min') # minute
Timestamp('2020-03-14 15:33:00')
- >>> ts.ceil(freq='s') # seconds
+ >>> ts.ceil(freq='s') # seconds
Timestamp('2020-03-14 15:32:53')
- >>> ts.ceil(freq='us') # microseconds
+ >>> ts.ceil(freq='us') # microseconds
Timestamp('2020-03-14 15:32:52.192549')
``freq`` can also be a multiple of a single unit, like '5min' (i.e. 5 minutes):
diff --git a/pandas/_libs/tslibs/tzconversion.pyx b/pandas/_libs/tslibs/tzconversion.pyx
index 2c4f0cd14db13..e3facd3d9599b 100644
--- a/pandas/_libs/tslibs/tzconversion.pyx
+++ b/pandas/_libs/tslibs/tzconversion.pyx
@@ -607,7 +607,8 @@ cdef ndarray[int64_t] _get_dst_hours(
ndarray[uint8_t, cast=True] mismatch
ndarray[int64_t] delta, dst_hours
ndarray[intp_t] switch_idxs, trans_idx, grp, a_idx, b_idx, one_diff
- list trans_grp
+ # TODO: Can uncomment when numpy >=2 is the minimum
+ # tuple trans_grp
intp_t switch_idx
int64_t left, right
diff --git a/pandas/_libs/tslibs/util.pxd b/pandas/_libs/tslibs/util.pxd
index e4ac3a9e167a3..a5822e57d3fa6 100644
--- a/pandas/_libs/tslibs/util.pxd
+++ b/pandas/_libs/tslibs/util.pxd
@@ -185,11 +185,11 @@ cdef inline const char* get_c_string(str py_string) except NULL:
return get_c_string_buf_and_size(py_string, NULL)
-cdef inline bytes string_encode_locale(str py_string) noexcept:
+cdef inline bytes string_encode_locale(str py_string):
"""As opposed to PyUnicode_Encode, use current system locale to encode."""
return PyUnicode_EncodeLocale(py_string, NULL)
-cdef inline object char_to_string_locale(const char* data) noexcept:
+cdef inline object char_to_string_locale(const char* data):
"""As opposed to PyUnicode_FromString, use current system locale to decode."""
return PyUnicode_DecodeLocale(data, NULL)
diff --git a/pandas/_libs/tslibs/vectorized.pyi b/pandas/_libs/tslibs/vectorized.pyi
index de19f592da62b..f377c2e26ab81 100644
--- a/pandas/_libs/tslibs/vectorized.pyi
+++ b/pandas/_libs/tslibs/vectorized.pyi
@@ -1,7 +1,5 @@
-"""
-For cython types that cannot be represented precisely, closest-available
-python equivalents are used, and the precise types kept as adjacent comments.
-"""
+# For cython types that cannot be represented precisely, closest-available
+# python equivalents are used, and the precise types kept as adjacent comments.
from datetime import tzinfo
import numpy as np
diff --git a/pandas/_testing/__init__.py b/pandas/_testing/__init__.py
index 672c16a85086c..12395b42bba19 100644
--- a/pandas/_testing/__init__.py
+++ b/pandas/_testing/__init__.py
@@ -34,7 +34,6 @@
Series,
)
from pandas._testing._io import (
- round_trip_localpath,
round_trip_pathlib,
round_trip_pickle,
write_to_compressed,
@@ -73,12 +72,10 @@
get_obj,
)
from pandas._testing.contexts import (
- assert_cow_warning,
decompress_file,
ensure_clean,
raises_chained_assignment_error,
set_timezone,
- use_numexpr,
with_csv_dialect,
)
from pandas.core.arrays import (
@@ -236,11 +233,18 @@
+ TIMEDELTA_PYARROW_DTYPES
+ BOOL_PYARROW_DTYPES
)
+ ALL_REAL_PYARROW_DTYPES_STR_REPR = (
+ ALL_INT_PYARROW_DTYPES_STR_REPR + FLOAT_PYARROW_DTYPES_STR_REPR
+ )
else:
FLOAT_PYARROW_DTYPES_STR_REPR = []
ALL_INT_PYARROW_DTYPES_STR_REPR = []
ALL_PYARROW_DTYPES = []
+ ALL_REAL_PYARROW_DTYPES_STR_REPR = []
+ALL_REAL_NULLABLE_DTYPES = (
+ FLOAT_NUMPY_DTYPES + ALL_REAL_EXTENSION_DTYPES + ALL_REAL_PYARROW_DTYPES_STR_REPR
+)
arithmetic_dunder_methods = [
"__add__",
@@ -394,9 +398,6 @@ def external_error_raised(expected_exception: type[Exception]) -> ContextManager
return pytest.raises(expected_exception, match=None)
-cython_table = pd.core.common._cython_table.items()
-
-
def get_cython_table_params(ndframe, func_names_and_expected):
"""
Combine frame, functions from com._cython_table
@@ -417,11 +418,6 @@ def get_cython_table_params(ndframe, func_names_and_expected):
results = []
for func_name, expected in func_names_and_expected:
results.append((ndframe, func_name, expected))
- results += [
- (ndframe, func, expected)
- for func, name in cython_table
- if name == func_name
- ]
return results
@@ -483,7 +479,7 @@ def iat(x):
_UNITS = ["s", "ms", "us", "ns"]
-def get_finest_unit(left: str, right: str):
+def get_finest_unit(left: str, right: str) -> str:
"""
Find the higher of two datetime64 units.
"""
@@ -577,7 +573,6 @@ def shares_memory(left, right) -> bool:
"assert_series_equal",
"assert_sp_array_equal",
"assert_timedelta_array_equal",
- "assert_cow_warning",
"at",
"BOOL_DTYPES",
"box_expected",
@@ -609,7 +604,6 @@ def shares_memory(left, right) -> bool:
"OBJECT_DTYPES",
"raise_assert_detail",
"raises_chained_assignment_error",
- "round_trip_localpath",
"round_trip_pathlib",
"round_trip_pickle",
"setitem",
@@ -625,7 +619,6 @@ def shares_memory(left, right) -> bool:
"to_array",
"UNSIGNED_INT_EA_DTYPES",
"UNSIGNED_INT_NUMPY_DTYPES",
- "use_numexpr",
"with_csv_dialect",
"write_to_compressed",
]
diff --git a/pandas/_testing/_hypothesis.py b/pandas/_testing/_hypothesis.py
index 084ca9c306d19..b7fc175b10d17 100644
--- a/pandas/_testing/_hypothesis.py
+++ b/pandas/_testing/_hypothesis.py
@@ -1,6 +1,7 @@
"""
Hypothesis data generator helpers.
"""
+
from datetime import datetime
from hypothesis import strategies as st
@@ -54,12 +55,8 @@
DATETIME_NO_TZ = st.datetimes()
DATETIME_JAN_1_1900_OPTIONAL_TZ = st.datetimes(
- min_value=pd.Timestamp(
- 1900, 1, 1
- ).to_pydatetime(), # pyright: ignore[reportGeneralTypeIssues]
- max_value=pd.Timestamp(
- 1900, 1, 1
- ).to_pydatetime(), # pyright: ignore[reportGeneralTypeIssues]
+ min_value=pd.Timestamp(1900, 1, 1).to_pydatetime(), # pyright: ignore[reportArgumentType]
+ max_value=pd.Timestamp(1900, 1, 1).to_pydatetime(), # pyright: ignore[reportArgumentType]
timezones=st.one_of(st.none(), dateutil_timezones(), pytz_timezones()),
)
diff --git a/pandas/_testing/_io.py b/pandas/_testing/_io.py
index 95977edb600ad..2955108d3db1a 100644
--- a/pandas/_testing/_io.py
+++ b/pandas/_testing/_io.py
@@ -89,35 +89,6 @@ def round_trip_pathlib(writer, reader, path: str | None = None):
return obj
-def round_trip_localpath(writer, reader, path: str | None = None):
- """
- Write an object to file specified by a py.path LocalPath and read it back.
-
- Parameters
- ----------
- writer : callable bound to pandas object
- IO writing function (e.g. DataFrame.to_csv )
- reader : callable
- IO reading function (e.g. pd.read_csv )
- path : str, default None
- The path where the object is written and then read.
-
- Returns
- -------
- pandas object
- The original object that was serialized and then re-read.
- """
- import pytest
-
- LocalPath = pytest.importorskip("py.path").local
- if path is None:
- path = "___localpath___"
- with ensure_clean(path) as path:
- writer(LocalPath(path))
- obj = reader(LocalPath(path))
- return obj
-
-
def write_to_compressed(compression, path, data, dest: str = "test") -> None:
"""
Write data to a compressed file.
diff --git a/pandas/_testing/_warnings.py b/pandas/_testing/_warnings.py
index f9cf390ba59de..d9516077788c8 100644
--- a/pandas/_testing/_warnings.py
+++ b/pandas/_testing/_warnings.py
@@ -76,10 +76,8 @@ class for all warnings. To raise multiple types of exceptions,
>>> import warnings
>>> with assert_produces_warning():
... warnings.warn(UserWarning())
- ...
>>> with assert_produces_warning(False):
... warnings.warn(RuntimeWarning())
- ...
Traceback (most recent call last):
...
AssertionError: Caused unexpected warning(s): ['RuntimeWarning'].
@@ -152,13 +150,12 @@ def _assert_caught_expected_warning(
if not saw_warning:
raise AssertionError(
- f"Did not see expected warning of class "
- f"{repr(expected_warning.__name__)}"
+ f"Did not see expected warning of class {expected_warning.__name__!r}"
)
if match and not matched_message:
raise AssertionError(
- f"Did not see warning {repr(expected_warning.__name__)} "
+ f"Did not see warning {expected_warning.__name__!r} "
f"matching '{match}'. The emitted warning messages are "
f"{unmatched_messages}"
)
@@ -200,7 +197,7 @@ def _assert_caught_no_extra_warnings(
)
if extra_warnings:
- raise AssertionError(f"Caused unexpected warning(s): {repr(extra_warnings)}")
+ raise AssertionError(f"Caused unexpected warning(s): {extra_warnings!r}")
def _is_unexpected_warning(
@@ -221,7 +218,12 @@ def _assert_raised_with_correct_stacklevel(
frame = inspect.currentframe()
for _ in range(4):
frame = frame.f_back # type: ignore[union-attr]
- caller_filename = inspect.getfile(frame) # type: ignore[arg-type]
+ try:
+ caller_filename = inspect.getfile(frame) # type: ignore[arg-type]
+ finally:
+ # See note in
+ # https://docs.python.org/3/library/inspect.html#inspect.Traceback
+ del frame
msg = (
"Warning not set with correct stacklevel. "
f"File where warning is raised: {actual_warning.filename} != "
diff --git a/pandas/_testing/asserters.py b/pandas/_testing/asserters.py
index d0f38c85868d4..3aacd3099c334 100644
--- a/pandas/_testing/asserters.py
+++ b/pandas/_testing/asserters.py
@@ -10,6 +10,7 @@
import numpy as np
+from pandas._libs import lib
from pandas._libs.missing import is_matching_na
from pandas._libs.sparse import SparseIndex
import pandas._libs.testing as _testing
@@ -431,14 +432,14 @@ def assert_is_valid_plot_return_object(objs) -> None:
for el in objs.ravel():
msg = (
"one of 'objs' is not a matplotlib Axes instance, "
- f"type encountered {repr(type(el).__name__)}"
+ f"type encountered {type(el).__name__!r}"
)
assert isinstance(el, (Axes, dict)), msg
else:
msg = (
"objs is neither an ndarray of Artist instances nor a single "
"ArtistArtist instance, tuple, or dict, 'objs' is a "
- f"{repr(type(objs).__name__)}"
+ f"{type(objs).__name__!r}"
)
assert isinstance(objs, (Artist, tuple, dict)), msg
@@ -660,10 +661,10 @@ def _get_base(obj):
if check_same == "same":
if left_base is not right_base:
- raise AssertionError(f"{repr(left_base)} is not {repr(right_base)}")
+ raise AssertionError(f"{left_base!r} is not {right_base!r}")
elif check_same == "copy":
if left_base is right_base:
- raise AssertionError(f"{repr(left_base)} is {repr(right_base)}")
+ raise AssertionError(f"{left_base!r} is {right_base!r}")
def _raise(left, right, err_msg) -> NoReturn:
if err_msg is None:
@@ -698,9 +699,9 @@ def assert_extension_array_equal(
right,
check_dtype: bool | Literal["equiv"] = True,
index_values=None,
- check_exact: bool = False,
- rtol: float = 1.0e-5,
- atol: float = 1.0e-8,
+ check_exact: bool | lib.NoDefault = lib.no_default,
+ rtol: float | lib.NoDefault = lib.no_default,
+ atol: float | lib.NoDefault = lib.no_default,
obj: str = "ExtensionArray",
) -> None:
"""
@@ -715,7 +716,12 @@ def assert_extension_array_equal(
index_values : Index | numpy.ndarray, default None
Optional index (shared by both left and right), used in output.
check_exact : bool, default False
- Whether to compare number exactly. Only takes effect for float dtypes.
+ Whether to compare number exactly.
+
+ .. versionchanged:: 2.2.0
+
+ Defaults to True for integer dtypes if none of
+ ``check_exact``, ``rtol`` and ``atol`` are specified.
rtol : float, default 1e-5
Relative tolerance. Only used when check_exact is False.
atol : float, default 1e-8
@@ -739,6 +745,23 @@ def assert_extension_array_equal(
>>> b, c = a.array, a.array
>>> tm.assert_extension_array_equal(b, c)
"""
+ if (
+ check_exact is lib.no_default
+ and rtol is lib.no_default
+ and atol is lib.no_default
+ ):
+ check_exact = (
+ is_numeric_dtype(left.dtype)
+ and not is_float_dtype(left.dtype)
+ or is_numeric_dtype(right.dtype)
+ and not is_float_dtype(right.dtype)
+ )
+ elif check_exact is lib.no_default:
+ check_exact = False
+
+ rtol = rtol if rtol is not lib.no_default else 1.0e-5
+ atol = atol if atol is not lib.no_default else 1.0e-8
+
assert isinstance(left, ExtensionArray), "left is not an ExtensionArray"
assert isinstance(right, ExtensionArray), "right is not an ExtensionArray"
if check_dtype:
@@ -784,10 +807,7 @@ def assert_extension_array_equal(
left_valid = left[~left_na].to_numpy(dtype=object)
right_valid = right[~right_na].to_numpy(dtype=object)
- if check_exact or (
- (is_numeric_dtype(left.dtype) and not is_float_dtype(left.dtype))
- or (is_numeric_dtype(right.dtype) and not is_float_dtype(right.dtype))
- ):
+ if check_exact:
assert_numpy_array_equal(
left_valid, right_valid, obj=obj, index_values=index_values
)
@@ -811,14 +831,14 @@ def assert_series_equal(
check_index_type: bool | Literal["equiv"] = "equiv",
check_series_type: bool = True,
check_names: bool = True,
- check_exact: bool = False,
+ check_exact: bool | lib.NoDefault = lib.no_default,
check_datetimelike_compat: bool = False,
check_categorical: bool = True,
check_category_order: bool = True,
check_freq: bool = True,
check_flags: bool = True,
- rtol: float = 1.0e-5,
- atol: float = 1.0e-8,
+ rtol: float | lib.NoDefault = lib.no_default,
+ atol: float | lib.NoDefault = lib.no_default,
obj: str = "Series",
*,
check_index: bool = True,
@@ -841,7 +861,12 @@ def assert_series_equal(
check_names : bool, default True
Whether to check the Series and Index names attribute.
check_exact : bool, default False
- Whether to compare number exactly. Only takes effect for float dtypes.
+ Whether to compare number exactly.
+
+ .. versionchanged:: 2.2.0
+
+ Defaults to True for integer dtypes if none of
+ ``check_exact``, ``rtol`` and ``atol`` are specified.
check_datetimelike_compat : bool, default False
Compare datetime-like which is comparable ignoring dtype.
check_categorical : bool, default True
@@ -877,6 +902,23 @@ def assert_series_equal(
>>> tm.assert_series_equal(a, b)
"""
__tracebackhide__ = True
+ check_exact_index = False if check_exact is lib.no_default else check_exact
+ if (
+ check_exact is lib.no_default
+ and rtol is lib.no_default
+ and atol is lib.no_default
+ ):
+ check_exact = (
+ is_numeric_dtype(left.dtype)
+ and not is_float_dtype(left.dtype)
+ or is_numeric_dtype(right.dtype)
+ and not is_float_dtype(right.dtype)
+ )
+ elif check_exact is lib.no_default:
+ check_exact = False
+
+ rtol = rtol if rtol is not lib.no_default else 1.0e-5
+ atol = atol if atol is not lib.no_default else 1.0e-8
if not check_index and check_like:
raise ValueError("check_like must be False if check_index is False")
@@ -894,7 +936,7 @@ def assert_series_equal(
raise_assert_detail(obj, "Series length are different", msg1, msg2)
if check_flags:
- assert left.flags == right.flags, f"{repr(left.flags)} != {repr(right.flags)}"
+ assert left.flags == right.flags, f"{left.flags!r} != {right.flags!r}"
if check_index:
# GH #38183
@@ -903,7 +945,7 @@ def assert_series_equal(
right.index,
exact=check_index_type,
check_names=check_names,
- check_exact=check_exact,
+ check_exact=check_exact_index,
check_categorical=check_categorical,
check_order=not check_like,
rtol=rtol,
@@ -931,10 +973,7 @@ def assert_series_equal(
pass
else:
assert_attr_equal("dtype", left, right, obj=f"Attributes of {obj}")
- if check_exact or (
- (is_numeric_dtype(left.dtype) and not is_float_dtype(left.dtype))
- or (is_numeric_dtype(right.dtype) and not is_float_dtype(right.dtype))
- ):
+ if check_exact:
left_values = left._values
right_values = right._values
# Only check exact if dtype is numeric
@@ -1061,14 +1100,14 @@ def assert_frame_equal(
check_frame_type: bool = True,
check_names: bool = True,
by_blocks: bool = False,
- check_exact: bool = False,
+ check_exact: bool | lib.NoDefault = lib.no_default,
check_datetimelike_compat: bool = False,
check_categorical: bool = True,
check_like: bool = False,
check_freq: bool = True,
check_flags: bool = True,
- rtol: float = 1.0e-5,
- atol: float = 1.0e-8,
+ rtol: float | lib.NoDefault = lib.no_default,
+ atol: float | lib.NoDefault = lib.no_default,
obj: str = "DataFrame",
) -> None:
"""
@@ -1103,7 +1142,12 @@ def assert_frame_equal(
Specify how to compare internal data. If False, compare by columns.
If True, compare by blocks.
check_exact : bool, default False
- Whether to compare number exactly. Only takes effect for float dtypes.
+ Whether to compare number exactly.
+
+ .. versionchanged:: 2.2.0
+
+ Defaults to True for integer dtypes if none of
+ ``check_exact``, ``rtol`` and ``atol`` are specified.
check_datetimelike_compat : bool, default False
Compare datetime-like which is comparable ignoring dtype.
check_categorical : bool, default True
@@ -1135,8 +1179,8 @@ def assert_frame_equal(
but with columns of differing dtypes.
>>> from pandas.testing import assert_frame_equal
- >>> df1 = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})
- >>> df2 = pd.DataFrame({'a': [1, 2], 'b': [3.0, 4.0]})
+ >>> df1 = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+ >>> df2 = pd.DataFrame({"a": [1, 2], "b": [3.0, 4.0]})
df1 equals itself.
@@ -1158,6 +1202,9 @@ def assert_frame_equal(
>>> assert_frame_equal(df1, df2, check_dtype=False)
"""
__tracebackhide__ = True
+ _rtol = rtol if rtol is not lib.no_default else 1.0e-5
+ _atol = atol if atol is not lib.no_default else 1.0e-8
+ _check_exact = check_exact if check_exact is not lib.no_default else False
# instance validation
_check_isinstance(left, right, DataFrame)
@@ -1169,11 +1216,11 @@ def assert_frame_equal(
# shape comparison
if left.shape != right.shape:
raise_assert_detail(
- obj, f"{obj} shape mismatch", f"{repr(left.shape)}", f"{repr(right.shape)}"
+ obj, f"{obj} shape mismatch", f"{left.shape!r}", f"{right.shape!r}"
)
if check_flags:
- assert left.flags == right.flags, f"{repr(left.flags)} != {repr(right.flags)}"
+ assert left.flags == right.flags, f"{left.flags!r} != {right.flags!r}"
# index comparison
assert_index_equal(
@@ -1181,11 +1228,11 @@ def assert_frame_equal(
right.index,
exact=check_index_type,
check_names=check_names,
- check_exact=check_exact,
+ check_exact=_check_exact,
check_categorical=check_categorical,
check_order=not check_like,
- rtol=rtol,
- atol=atol,
+ rtol=_rtol,
+ atol=_atol,
obj=f"{obj}.index",
)
@@ -1195,11 +1242,11 @@ def assert_frame_equal(
right.columns,
exact=check_column_type,
check_names=check_names,
- check_exact=check_exact,
+ check_exact=_check_exact,
check_categorical=check_categorical,
check_order=not check_like,
- rtol=rtol,
- atol=atol,
+ rtol=_rtol,
+ atol=_atol,
obj=f"{obj}.columns",
)
@@ -1323,7 +1370,7 @@ def assert_sp_array_equal(left, right) -> None:
def assert_contains_all(iterable, dic) -> None:
for k in iterable:
- assert k in dic, f"Did not contain item: {repr(k)}"
+ assert k in dic, f"Did not contain item: {k!r}"
def assert_copy(iter1, iter2, **eql_kwargs) -> None:
@@ -1338,7 +1385,7 @@ def assert_copy(iter1, iter2, **eql_kwargs) -> None:
for elem1, elem2 in zip(iter1, iter2):
assert_almost_equal(elem1, elem2, **eql_kwargs)
msg = (
- f"Expected object {repr(type(elem1))} and object {repr(type(elem2))} to be "
+ f"Expected object {type(elem1)!r} and object {type(elem2)!r} to be "
"different objects, but they were the same object."
)
assert elem1 is not elem2, msg
diff --git a/pandas/_testing/compat.py b/pandas/_testing/compat.py
index cc352ba7b8f2f..722ba61a3227f 100644
--- a/pandas/_testing/compat.py
+++ b/pandas/_testing/compat.py
@@ -1,6 +1,7 @@
"""
Helpers for sharing tests between DataFrame/Series
"""
+
from __future__ import annotations
from typing import TYPE_CHECKING
diff --git a/pandas/_testing/contexts.py b/pandas/_testing/contexts.py
index eb6e4a917889a..b986e03e25815 100644
--- a/pandas/_testing/contexts.py
+++ b/pandas/_testing/contexts.py
@@ -16,8 +16,6 @@
from pandas.compat import PYPY
from pandas.errors import ChainedAssignmentError
-from pandas import set_option
-
from pandas.io.common import get_handle
if TYPE_CHECKING:
@@ -70,9 +68,8 @@ def set_timezone(tz: str) -> Generator[None, None, None]:
>>> tzlocal().tzname(datetime(2021, 1, 1)) # doctest: +SKIP
'IST'
- >>> with set_timezone('US/Eastern'):
+ >>> with set_timezone("US/Eastern"):
... tzlocal().tzname(datetime(2021, 1, 1))
- ...
'EST'
"""
import time
@@ -96,9 +93,7 @@ def setTZ(tz) -> None:
@contextmanager
-def ensure_clean(
- filename=None, return_filelike: bool = False, **kwargs: Any
-) -> Generator[Any, None, None]:
+def ensure_clean(filename=None) -> Generator[Any, None, None]:
"""
Gets a temporary path and agrees to remove on close.
@@ -110,12 +105,6 @@ def ensure_clean(
----------
filename : str (optional)
suffix of the created file.
- return_filelike : bool (default False)
- if True, returns a file-like which is *always* cleaned. Necessary for
- savefig and other functions which want to append extensions.
- **kwargs
- Additional keywords are passed to open().
-
"""
folder = Path(tempfile.gettempdir())
@@ -126,19 +115,11 @@ def ensure_clean(
path.touch()
- handle_or_str: str | IO = str(path)
- encoding = kwargs.pop("encoding", None)
- if return_filelike:
- kwargs.setdefault("mode", "w+b")
- if encoding is None and "b" not in kwargs["mode"]:
- encoding = "utf-8"
- handle_or_str = open(path, encoding=encoding, **kwargs)
+ handle_or_str = str(path)
try:
yield handle_or_str
finally:
- if not isinstance(handle_or_str, str):
- handle_or_str.close()
if path.is_file():
path.unlink()
@@ -177,24 +158,6 @@ def with_csv_dialect(name: str, **kwargs) -> Generator[None, None, None]:
csv.unregister_dialect(name)
-@contextmanager
-def use_numexpr(use, min_elements=None) -> Generator[None, None, None]:
- from pandas.core.computation import expressions as expr
-
- if min_elements is None:
- min_elements = expr._MIN_ELEMENTS
-
- olduse = expr.USE_NUMEXPR
- oldmin = expr._MIN_ELEMENTS
- set_option("compute.use_numexpr", use)
- expr._MIN_ELEMENTS = min_elements
- try:
- yield
- finally:
- expr._MIN_ELEMENTS = oldmin
- set_option("compute.use_numexpr", olduse)
-
-
def raises_chained_assignment_error(warn=True, extra_warnings=(), extra_match=()):
from pandas._testing import assert_produces_warning
@@ -229,29 +192,3 @@ def raises_chained_assignment_error(warn=True, extra_warnings=(), extra_match=()
warning,
match="|".join((match, *extra_match)),
)
-
-
-def assert_cow_warning(warn=True, match=None, **kwargs):
- """
- Assert that a warning is raised in the CoW warning mode.
-
- Parameters
- ----------
- warn : bool, default True
- By default, check that a warning is raised. Can be turned off by passing False.
- match : str
- The warning message to match against, if different from the default.
- kwargs
- Passed through to assert_produces_warning
- """
- from pandas._testing import assert_produces_warning
-
- if not warn:
- from contextlib import nullcontext
-
- return nullcontext()
-
- if not match:
- match = "Setting a value on a view"
-
- return assert_produces_warning(FutureWarning, match=match, **kwargs)
diff --git a/pandas/_typing.py b/pandas/_typing.py
index 3df9a47a35fca..f868a92554b39 100644
--- a/pandas/_typing.py
+++ b/pandas/_typing.py
@@ -46,7 +46,12 @@
from pandas.core.dtypes.dtypes import ExtensionDtype
- from pandas import Interval
+ from pandas import (
+ DatetimeIndex,
+ Interval,
+ PeriodIndex,
+ TimedeltaIndex,
+ )
from pandas.arrays import (
DatetimeArray,
TimedeltaArray,
@@ -61,9 +66,7 @@
)
from pandas.core.indexes.base import Index
from pandas.core.internals import (
- ArrayManager,
BlockManager,
- SingleArrayManager,
SingleBlockManager,
)
from pandas.core.resample import Resampler
@@ -90,25 +93,41 @@
from typing import SupportsIndex
if sys.version_info >= (3, 10):
+ from typing import Concatenate # pyright: ignore[reportUnusedImport]
+ from typing import ParamSpec
from typing import TypeGuard # pyright: ignore[reportUnusedImport]
else:
- from typing_extensions import TypeGuard # pyright: ignore[reportUnusedImport]
+ from typing_extensions import ( # pyright: ignore[reportUnusedImport]
+ Concatenate,
+ ParamSpec,
+ TypeGuard,
+ )
+
+ P = ParamSpec("P")
if sys.version_info >= (3, 11):
from typing import Self # pyright: ignore[reportUnusedImport]
+ from typing import Unpack # pyright: ignore[reportUnusedImport]
else:
from typing_extensions import Self # pyright: ignore[reportUnusedImport]
+ from typing_extensions import Unpack # pyright: ignore[reportUnusedImport]
+
else:
npt: Any = None
+ ParamSpec: Any = None
Self: Any = None
TypeGuard: Any = None
+ Concatenate: Any = None
+ Unpack: Any = None
HashableT = TypeVar("HashableT", bound=Hashable)
+HashableT2 = TypeVar("HashableT2", bound=Hashable)
MutableMappingT = TypeVar("MutableMappingT", bound=MutableMapping)
# array-like
ArrayLike = Union["ExtensionArray", np.ndarray]
+ArrayLikeT = TypeVar("ArrayLikeT", "ExtensionArray", np.ndarray)
AnyArrayLike = Union[ArrayLike, "Index", "Series"]
TimeArrayLike = Union["DatetimeArray", "TimedeltaArray"]
@@ -121,30 +140,22 @@
class SequenceNotStr(Protocol[_T_co]):
@overload
- def __getitem__(self, index: SupportsIndex, /) -> _T_co:
- ...
+ def __getitem__(self, index: SupportsIndex, /) -> _T_co: ...
@overload
- def __getitem__(self, index: slice, /) -> Sequence[_T_co]:
- ...
+ def __getitem__(self, index: slice, /) -> Sequence[_T_co]: ...
- def __contains__(self, value: object, /) -> bool:
- ...
+ def __contains__(self, value: object, /) -> bool: ...
- def __len__(self) -> int:
- ...
+ def __len__(self) -> int: ...
- def __iter__(self) -> Iterator[_T_co]:
- ...
+ def __iter__(self) -> Iterator[_T_co]: ...
- def index(self, value: Any, /, start: int = 0, stop: int = ...) -> int:
- ...
+ def index(self, value: Any, start: int = ..., stop: int = ..., /) -> int: ...
- def count(self, value: Any, /) -> int:
- ...
+ def count(self, value: Any, /) -> int: ...
- def __reversed__(self) -> Iterator[_T_co]:
- ...
+ def __reversed__(self) -> Iterator[_T_co]: ...
ListLike = Union[AnyArrayLike, SequenceNotStr, range]
@@ -179,6 +190,8 @@ def __reversed__(self) -> Iterator[_T_co]:
# passed in, a DataFrame is always returned.
NDFrameT = TypeVar("NDFrameT", bound="NDFrame")
+IndexT = TypeVar("IndexT", bound="Index")
+FreqIndexT = TypeVar("FreqIndexT", "DatetimeIndex", "PeriodIndex", "TimedeltaIndex")
NumpyIndexT = TypeVar("NumpyIndexT", np.ndarray, "Index")
AxisInt = int
@@ -186,7 +199,7 @@ def __reversed__(self) -> Iterator[_T_co]:
IndexLabel = Union[Hashable, Sequence[Hashable]]
Level = Hashable
Shape = tuple[int, ...]
-Suffixes = tuple[Optional[str], Optional[str]]
+Suffixes = Sequence[Optional[str]]
Ordered = Optional[bool]
JSONSerializable = Optional[Union[PythonScalar, list, dict]]
Frequency = Union[str, "BaseOffset"]
@@ -205,7 +218,7 @@ def __reversed__(self) -> Iterator[_T_co]:
Dtype = Union["ExtensionDtype", NpDtype]
AstypeArg = Union["ExtensionDtype", "npt.DTypeLike"]
# DtypeArg specifies all allowable dtypes in a functions its dtype argument
-DtypeArg = Union[Dtype, dict[Hashable, Dtype]]
+DtypeArg = Union[Dtype, Mapping[Hashable, Dtype]]
DtypeObj = Union[np.dtype, "ExtensionDtype"]
# converters
@@ -296,13 +309,11 @@ def flush(self) -> Any:
class ReadPickleBuffer(ReadBuffer[bytes], Protocol):
- def readline(self) -> bytes:
- ...
+ def readline(self) -> bytes: ...
class WriteExcelBuffer(WriteBuffer[bytes], Protocol):
- def truncate(self, size: int | None = ...) -> int:
- ...
+ def truncate(self, size: int | None = ...) -> int: ...
class ReadCsvBuffer(ReadBuffer[AnyStr_co], Protocol):
@@ -320,7 +331,7 @@ def readline(self) -> AnyStr_co:
@property
def closed(self) -> bool:
- # for enine=pyarrow
+ # for engine=pyarrow
...
@@ -370,11 +381,7 @@ def closed(self) -> bool:
]
# internals
-Manager = Union[
- "ArrayManager", "SingleArrayManager", "BlockManager", "SingleBlockManager"
-]
-SingleManager = Union["SingleArrayManager", "SingleBlockManager"]
-Manager2D = Union["ArrayManager", "BlockManager"]
+Manager = Union["BlockManager", "SingleBlockManager"]
# indexing
# PositionalIndexer -> valid 1D positional indexer, e.g. can pass
@@ -421,7 +428,7 @@ def closed(self) -> bool:
# datetime and NaTType
DatetimeNaTType = Union[datetime, "NaTType"]
-DateTimeErrorChoices = Union[IgnoreRaise, Literal["coerce"]]
+DateTimeErrorChoices = Literal["raise", "coerce"]
# sort_index
SortKind = Literal["quicksort", "mergesort", "heapsort", "stable"]
@@ -503,9 +510,6 @@ def closed(self) -> bool:
# from_dict
FromDictOrient = Literal["columns", "index", "tight"]
-# to_gbc
-ToGbqIfexist = Literal["fail", "replace", "append"]
-
# to_stata
ToStataByteorder = Literal[">", "<", "little", "big"]
@@ -523,3 +527,6 @@ def closed(self) -> bool:
Callable[[HashableT], bool],
None,
]
+
+# maintaine the sub-type of any hashable sequence
+SequenceT = TypeVar("SequenceT", bound=Sequence[Hashable])
diff --git a/pandas/_version.py b/pandas/_version.py
index f8a960630126d..7bd9da2bb1cfa 100644
--- a/pandas/_version.py
+++ b/pandas/_version.py
@@ -141,7 +141,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose):
if verbose:
print(
- f"Tried directories {str(rootdirs)} \
+ f"Tried directories {rootdirs!s} \
but none started with prefix {parentdir_prefix}"
)
raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
@@ -358,9 +358,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command):
if verbose:
fmt = "tag '%s' doesn't start with prefix '%s'"
print(fmt % (full_tag, tag_prefix))
- pieces[
- "error"
- ] = f"tag '{full_tag}' doesn't start with prefix '{tag_prefix}'"
+ pieces["error"] = (
+ f"tag '{full_tag}' doesn't start with prefix '{tag_prefix}'"
+ )
return pieces
pieces["closest-tag"] = full_tag[len(tag_prefix) :]
diff --git a/pandas/api/__init__.py b/pandas/api/__init__.py
index a0d42b6541fdf..9b007e8fe8da4 100644
--- a/pandas/api/__init__.py
+++ b/pandas/api/__init__.py
@@ -1,4 +1,5 @@
-""" public toolkit API """
+"""public toolkit API"""
+
from pandas.api import (
extensions,
indexers,
diff --git a/pandas/arrays/__init__.py b/pandas/arrays/__init__.py
index a11755275d00e..bcf295fd6b490 100644
--- a/pandas/arrays/__init__.py
+++ b/pandas/arrays/__init__.py
@@ -3,6 +3,7 @@
See :ref:`extending.extension-types` for more.
"""
+
from pandas.core.arrays import (
ArrowExtensionArray,
ArrowStringArray,
diff --git a/pandas/compat/__init__.py b/pandas/compat/__init__.py
index 738442fab8c70..caa00b205a29c 100644
--- a/pandas/compat/__init__.py
+++ b/pandas/compat/__init__.py
@@ -7,6 +7,7 @@
Other items:
* platform checker
"""
+
from __future__ import annotations
import os
@@ -30,6 +31,7 @@
pa_version_under13p0,
pa_version_under14p0,
pa_version_under14p1,
+ pa_version_under16p0,
)
if TYPE_CHECKING:
@@ -186,6 +188,7 @@ def get_bz2_file() -> type[pandas.compat.compressors.BZ2File]:
"pa_version_under13p0",
"pa_version_under14p0",
"pa_version_under14p1",
+ "pa_version_under16p0",
"IS64",
"ISMUSL",
"PY310",
diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py
index 3cdeae52a25ba..f4e717c26d6fd 100644
--- a/pandas/compat/_optional.py
+++ b/pandas/compat/_optional.py
@@ -16,16 +16,16 @@
if TYPE_CHECKING:
import types
-# Update install.rst & setup.cfg when updating versions!
+# Update install.rst, actions-39-minimum_versions.yaml,
+# deps_minimum.toml & pyproject.toml when updating versions!
VERSIONS = {
- "adbc-driver-postgresql": "0.8.0",
+ "adbc-driver-postgresql": "0.10.0",
"adbc-driver-sqlite": "0.8.0",
"bs4": "4.11.2",
"blosc": "1.21.3",
"bottleneck": "1.3.6",
- "dataframe-api-compat": "0.1.7",
- "fastparquet": "2022.12.0",
+ "fastparquet": "2023.10.0",
"fsspec": "2022.11.0",
"html5lib": "1.1",
"hypothesis": "6.46.1",
@@ -37,7 +37,6 @@
"numexpr": "2.8.4",
"odfpy": "1.4.1",
"openpyxl": "3.1.0",
- "pandas_gbq": "0.19.0",
"psycopg2": "2.9.6", # (dt dec pq3 ext lo64)
"pymysql": "1.0.2",
"pyarrow": "10.0.1",
@@ -68,7 +67,6 @@
"jinja2": "Jinja2",
"lxml.etree": "lxml",
"odf": "odfpy",
- "pandas_gbq": "pandas-gbq",
"python_calamine": "python-calamine",
"sqlalchemy": "SQLAlchemy",
"tables": "pytables",
@@ -93,8 +91,7 @@ def import_optional_dependency(
min_version: str | None = ...,
*,
errors: Literal["raise"] = ...,
-) -> types.ModuleType:
- ...
+) -> types.ModuleType: ...
@overload
@@ -104,8 +101,7 @@ def import_optional_dependency(
min_version: str | None = ...,
*,
errors: Literal["warn", "ignore"],
-) -> types.ModuleType | None:
- ...
+) -> types.ModuleType | None: ...
def import_optional_dependency(
@@ -147,9 +143,8 @@ def import_optional_dependency(
The imported module, when found and the version is correct.
None is returned when the package is not found and `errors`
is False, or when the package's version is too old and `errors`
- is ``'warn'``.
+ is ``'warn'`` or ``'ignore'``.
"""
-
assert errors in {"warn", "raise", "ignore"}
package_name = INSTALL_MAPPING.get(name)
@@ -161,9 +156,9 @@ def import_optional_dependency(
)
try:
module = importlib.import_module(name)
- except ImportError:
+ except ImportError as err:
if errors == "raise":
- raise ImportError(msg)
+ raise ImportError(msg) from err
return None
# Handle submodules: if we have submodule, grab parent module from sys.modules
@@ -190,5 +185,7 @@ def import_optional_dependency(
return None
elif errors == "raise":
raise ImportError(msg)
+ else:
+ return None
return module
diff --git a/pandas/compat/numpy/__init__.py b/pandas/compat/numpy/__init__.py
index 3014bd652d8c4..54a12c76a230b 100644
--- a/pandas/compat/numpy/__init__.py
+++ b/pandas/compat/numpy/__init__.py
@@ -1,4 +1,5 @@
-""" support numpy compatibility across versions """
+"""support numpy compatibility across versions"""
+
import warnings
import numpy as np
@@ -8,13 +9,12 @@
# numpy versioning
_np_version = np.__version__
_nlv = Version(_np_version)
-np_version_lt1p23 = _nlv < Version("1.23")
np_version_gte1p24 = _nlv >= Version("1.24")
np_version_gte1p24p3 = _nlv >= Version("1.24.3")
np_version_gte1p25 = _nlv >= Version("1.25")
np_version_gt2 = _nlv >= Version("2.0.0.dev0")
is_numpy_dev = _nlv.dev is not None
-_min_numpy_ver = "1.22.4"
+_min_numpy_ver = "1.23.5"
if _nlv < Version(_min_numpy_ver):
diff --git a/pandas/compat/numpy/function.py b/pandas/compat/numpy/function.py
index a36e25a9df410..abf86fc415641 100644
--- a/pandas/compat/numpy/function.py
+++ b/pandas/compat/numpy/function.py
@@ -15,6 +15,7 @@
methods that are spread throughout the codebase. This module will make it
easier to adjust to future upstream changes in the analogous numpy signatures.
"""
+
from __future__ import annotations
from typing import (
@@ -138,6 +139,7 @@ def validate_argmax_with_skipna(skipna: bool | ndarray | None, args, kwargs) ->
ARGSORT_DEFAULTS["kind"] = "quicksort"
ARGSORT_DEFAULTS["order"] = None
ARGSORT_DEFAULTS["kind"] = None
+ARGSORT_DEFAULTS["stable"] = None
validate_argsort = CompatValidator(
@@ -149,6 +151,7 @@ def validate_argmax_with_skipna(skipna: bool | ndarray | None, args, kwargs) ->
ARGSORT_DEFAULTS_KIND: dict[str, int | None] = {}
ARGSORT_DEFAULTS_KIND["axis"] = -1
ARGSORT_DEFAULTS_KIND["order"] = None
+ARGSORT_DEFAULTS_KIND["stable"] = None
validate_argsort_kind = CompatValidator(
ARGSORT_DEFAULTS_KIND, fname="argsort", max_fname_arg_count=0, method="both"
)
@@ -177,13 +180,11 @@ def validate_argsort_with_ascending(ascending: bool | int | None, args, kwargs)
@overload
-def validate_clip_with_axis(axis: ndarray, args, kwargs) -> None:
- ...
+def validate_clip_with_axis(axis: ndarray, args, kwargs) -> None: ...
@overload
-def validate_clip_with_axis(axis: AxisNoneT, args, kwargs) -> AxisNoneT:
- ...
+def validate_clip_with_axis(axis: AxisNoneT, args, kwargs) -> AxisNoneT: ...
def validate_clip_with_axis(
@@ -257,10 +258,6 @@ def validate_cum_func_with_skipna(skipna: bool, args, kwargs, name) -> bool:
MINMAX_DEFAULTS, fname="max", method="both", max_fname_arg_count=1
)
-RESHAPE_DEFAULTS: dict[str, str] = {"order": "C"}
-validate_reshape = CompatValidator(
- RESHAPE_DEFAULTS, fname="reshape", method="both", max_fname_arg_count=1
-)
REPEAT_DEFAULTS: dict[str, Any] = {"axis": None}
validate_repeat = CompatValidator(
@@ -272,12 +269,6 @@ def validate_cum_func_with_skipna(skipna: bool, args, kwargs, name) -> bool:
ROUND_DEFAULTS, fname="round", method="both", max_fname_arg_count=1
)
-SORT_DEFAULTS: dict[str, int | str | None] = {}
-SORT_DEFAULTS["axis"] = -1
-SORT_DEFAULTS["kind"] = "quicksort"
-SORT_DEFAULTS["order"] = None
-validate_sort = CompatValidator(SORT_DEFAULTS, fname="sort", method="kwargs")
-
STAT_FUNC_DEFAULTS: dict[str, Any | None] = {}
STAT_FUNC_DEFAULTS["dtype"] = None
STAT_FUNC_DEFAULTS["out"] = None
@@ -323,20 +314,6 @@ def validate_cum_func_with_skipna(skipna: bool, args, kwargs, name) -> bool:
validate_take = CompatValidator(TAKE_DEFAULTS, fname="take", method="kwargs")
-def validate_take_with_convert(convert: ndarray | bool | None, args, kwargs) -> bool:
- """
- If this function is called via the 'numpy' library, the third parameter in
- its signature is 'axis', which takes either an ndarray or 'None', so check
- if the 'convert' parameter is either an instance of ndarray or is None
- """
- if isinstance(convert, ndarray) or convert is None:
- args = (convert,) + args
- convert = True
-
- validate_take(args, kwargs, max_fname_arg_count=3, method="both")
- return convert
-
-
TRANSPOSE_DEFAULTS = {"axes": None}
validate_transpose = CompatValidator(
TRANSPOSE_DEFAULTS, fname="transpose", method="both", max_fname_arg_count=0
@@ -361,23 +338,6 @@ def validate_groupby_func(name: str, args, kwargs, allowed=None) -> None:
)
-RESAMPLER_NUMPY_OPS = ("min", "max", "sum", "prod", "mean", "std", "var")
-
-
-def validate_resampler_func(method: str, args, kwargs) -> None:
- """
- 'args' and 'kwargs' should be empty because all of their necessary
- parameters are explicitly listed in the function signature
- """
- if len(args) + len(kwargs) > 0:
- if method in RESAMPLER_NUMPY_OPS:
- raise UnsupportedFunctionCall(
- "numpy operations are not valid with resample. "
- f"Use .resample(...).{method}() instead"
- )
- raise TypeError("too many arguments passed in")
-
-
def validate_minmax_axis(axis: AxisInt | None, ndim: int = 1) -> None:
"""
Ensure that the axis argument passed to min, max, argmin, or argmax is zero
diff --git a/pandas/compat/pickle_compat.py b/pandas/compat/pickle_compat.py
index cd98087c06c18..28985a1380bee 100644
--- a/pandas/compat/pickle_compat.py
+++ b/pandas/compat/pickle_compat.py
@@ -1,20 +1,22 @@
"""
-Support pre-0.12 series pickle compatibility.
+Pickle compatibility to pandas version 1.0
"""
+
from __future__ import annotations
import contextlib
-import copy
import io
-import pickle as pkl
-from typing import TYPE_CHECKING
+import pickle
+from typing import (
+ TYPE_CHECKING,
+ Any,
+)
import numpy as np
from pandas._libs.arrays import NDArrayBacked
from pandas._libs.tslibs import BaseOffset
-from pandas import Index
from pandas.core.arrays import (
DatetimeArray,
PeriodArray,
@@ -26,111 +28,20 @@
from collections.abc import Generator
-def load_reduce(self) -> None:
- stack = self.stack
- args = stack.pop()
- func = stack[-1]
-
- try:
- stack[-1] = func(*args)
- return
- except TypeError as err:
- # If we have a deprecated function,
- # try to replace and try again.
-
- msg = "_reconstruct: First argument must be a sub-type of ndarray"
-
- if msg in str(err):
- try:
- cls = args[0]
- stack[-1] = object.__new__(cls)
- return
- except TypeError:
- pass
- elif args and isinstance(args[0], type) and issubclass(args[0], BaseOffset):
- # TypeError: object.__new__(Day) is not safe, use Day.__new__()
- cls = args[0]
- stack[-1] = cls.__new__(*args)
- return
- elif args and issubclass(args[0], PeriodArray):
- cls = args[0]
- stack[-1] = NDArrayBacked.__new__(*args)
- return
-
- raise
-
-
# If classes are moved, provide compat here.
_class_locations_map = {
- ("pandas.core.sparse.array", "SparseArray"): ("pandas.core.arrays", "SparseArray"),
- # 15477
- ("pandas.core.base", "FrozenNDArray"): ("numpy", "ndarray"),
# Re-routing unpickle block logic to go through _unpickle_block instead
# for pandas <= 1.3.5
("pandas.core.internals.blocks", "new_block"): (
"pandas._libs.internals",
"_unpickle_block",
),
- ("pandas.core.indexes.frozen", "FrozenNDArray"): ("numpy", "ndarray"),
- ("pandas.core.base", "FrozenList"): ("pandas.core.indexes.frozen", "FrozenList"),
- # 10890
- ("pandas.core.series", "TimeSeries"): ("pandas.core.series", "Series"),
- ("pandas.sparse.series", "SparseTimeSeries"): (
- "pandas.core.sparse.series",
- "SparseSeries",
- ),
- # 12588, extensions moving
- ("pandas._sparse", "BlockIndex"): ("pandas._libs.sparse", "BlockIndex"),
- ("pandas.tslib", "Timestamp"): ("pandas._libs.tslib", "Timestamp"),
- # 18543 moving period
- ("pandas._period", "Period"): ("pandas._libs.tslibs.period", "Period"),
- ("pandas._libs.period", "Period"): ("pandas._libs.tslibs.period", "Period"),
- # 18014 moved __nat_unpickle from _libs.tslib-->_libs.tslibs.nattype
- ("pandas.tslib", "__nat_unpickle"): (
+ # Avoid Cython's warning "contradiction to to Python 'class private name' rules"
+ ("pandas._libs.tslibs.nattype", "__nat_unpickle"): (
"pandas._libs.tslibs.nattype",
- "__nat_unpickle",
- ),
- ("pandas._libs.tslib", "__nat_unpickle"): (
- "pandas._libs.tslibs.nattype",
- "__nat_unpickle",
- ),
- # 15998 top-level dirs moving
- ("pandas.sparse.array", "SparseArray"): (
- "pandas.core.arrays.sparse",
- "SparseArray",
- ),
- ("pandas.indexes.base", "_new_Index"): ("pandas.core.indexes.base", "_new_Index"),
- ("pandas.indexes.base", "Index"): ("pandas.core.indexes.base", "Index"),
- ("pandas.indexes.numeric", "Int64Index"): (
- "pandas.core.indexes.base",
- "Index", # updated in 50775
- ),
- ("pandas.indexes.range", "RangeIndex"): ("pandas.core.indexes.range", "RangeIndex"),
- ("pandas.indexes.multi", "MultiIndex"): ("pandas.core.indexes.multi", "MultiIndex"),
- ("pandas.tseries.index", "_new_DatetimeIndex"): (
- "pandas.core.indexes.datetimes",
- "_new_DatetimeIndex",
- ),
- ("pandas.tseries.index", "DatetimeIndex"): (
- "pandas.core.indexes.datetimes",
- "DatetimeIndex",
- ),
- ("pandas.tseries.period", "PeriodIndex"): (
- "pandas.core.indexes.period",
- "PeriodIndex",
- ),
- # 19269, arrays moving
- ("pandas.core.categorical", "Categorical"): ("pandas.core.arrays", "Categorical"),
- # 19939, add timedeltaindex, float64index compat from 15998 move
- ("pandas.tseries.tdi", "TimedeltaIndex"): (
- "pandas.core.indexes.timedeltas",
- "TimedeltaIndex",
- ),
- ("pandas.indexes.numeric", "Float64Index"): (
- "pandas.core.indexes.base",
- "Index", # updated in 50775
+ "_nat_unpickle",
),
- # 50775, remove Int64Index, UInt64Index & Float64Index from codabase
+ # 50775, remove Int64Index, UInt64Index & Float64Index from codebase
("pandas.core.indexes.numeric", "Int64Index"): (
"pandas.core.indexes.base",
"Index",
@@ -152,85 +63,55 @@ def load_reduce(self) -> None:
# our Unpickler sub-class to override methods and some dispatcher
# functions for compat and uses a non-public class of the pickle module.
-
-
-class Unpickler(pkl._Unpickler):
- def find_class(self, module, name):
- # override superclass
+class Unpickler(pickle._Unpickler):
+ def find_class(self, module: str, name: str) -> Any:
key = (module, name)
module, name = _class_locations_map.get(key, key)
return super().find_class(module, name)
+ dispatch = pickle._Unpickler.dispatch.copy()
-Unpickler.dispatch = copy.copy(Unpickler.dispatch)
-Unpickler.dispatch[pkl.REDUCE[0]] = load_reduce
-
-
-def load_newobj(self) -> None:
- args = self.stack.pop()
- cls = self.stack[-1]
-
- # compat
- if issubclass(cls, Index):
- obj = object.__new__(cls)
- elif issubclass(cls, DatetimeArray) and not args:
- arr = np.array([], dtype="M8[ns]")
- obj = cls.__new__(cls, arr, arr.dtype)
- elif issubclass(cls, TimedeltaArray) and not args:
- arr = np.array([], dtype="m8[ns]")
- obj = cls.__new__(cls, arr, arr.dtype)
- elif cls is BlockManager and not args:
- obj = cls.__new__(cls, (), [], False)
- else:
- obj = cls.__new__(cls, *args)
-
- self.stack[-1] = obj
-
-
-Unpickler.dispatch[pkl.NEWOBJ[0]] = load_newobj
-
-
-def load_newobj_ex(self) -> None:
- kwargs = self.stack.pop()
- args = self.stack.pop()
- cls = self.stack.pop()
-
- # compat
- if issubclass(cls, Index):
- obj = object.__new__(cls)
- else:
- obj = cls.__new__(cls, *args, **kwargs)
- self.append(obj)
+ def load_reduce(self) -> None:
+ stack = self.stack # type: ignore[attr-defined]
+ args = stack.pop()
+ func = stack[-1]
-
-try:
- Unpickler.dispatch[pkl.NEWOBJ_EX[0]] = load_newobj_ex
-except (AttributeError, KeyError):
- pass
-
-
-def load(fh, encoding: str | None = None, is_verbose: bool = False):
- """
- Load a pickle, with a provided encoding,
-
- Parameters
- ----------
- fh : a filelike object
- encoding : an optional encoding
- is_verbose : show exception output
- """
- try:
- fh.seek(0)
- if encoding is not None:
- up = Unpickler(fh, encoding=encoding)
+ try:
+ stack[-1] = func(*args)
+ except TypeError:
+ # If we have a deprecated function,
+ # try to replace and try again.
+ if args and isinstance(args[0], type) and issubclass(args[0], BaseOffset):
+ # TypeError: object.__new__(Day) is not safe, use Day.__new__()
+ cls = args[0]
+ stack[-1] = cls.__new__(*args)
+ return
+ elif args and issubclass(args[0], PeriodArray):
+ cls = args[0]
+ stack[-1] = NDArrayBacked.__new__(*args)
+ return
+ raise
+
+ dispatch[pickle.REDUCE[0]] = load_reduce # type: ignore[assignment]
+
+ def load_newobj(self) -> None:
+ args = self.stack.pop() # type: ignore[attr-defined]
+ cls = self.stack.pop() # type: ignore[attr-defined]
+
+ # compat
+ if issubclass(cls, DatetimeArray) and not args:
+ arr = np.array([], dtype="M8[ns]")
+ obj = cls.__new__(cls, arr, arr.dtype)
+ elif issubclass(cls, TimedeltaArray) and not args:
+ arr = np.array([], dtype="m8[ns]")
+ obj = cls.__new__(cls, arr, arr.dtype)
+ elif cls is BlockManager and not args:
+ obj = cls.__new__(cls, (), [], False)
else:
- up = Unpickler(fh)
- # "Unpickler" has no attribute "is_verbose" [attr-defined]
- up.is_verbose = is_verbose # type: ignore[attr-defined]
+ obj = cls.__new__(cls, *args)
+ self.append(obj) # type: ignore[attr-defined]
- return up.load()
- except (ValueError, TypeError):
- raise
+ dispatch[pickle.NEWOBJ[0]] = load_newobj # type: ignore[assignment]
def loads(
@@ -239,7 +120,7 @@ def loads(
fix_imports: bool = True,
encoding: str = "ASCII",
errors: str = "strict",
-):
+) -> Any:
"""
Analogous to pickle._loads.
"""
@@ -254,9 +135,9 @@ def patch_pickle() -> Generator[None, None, None]:
"""
Temporarily patch pickle to use our unpickler.
"""
- orig_loads = pkl.loads
+ orig_loads = pickle.loads
try:
- setattr(pkl, "loads", loads)
+ setattr(pickle, "loads", loads)
yield
finally:
- setattr(pkl, "loads", orig_loads)
+ setattr(pickle, "loads", orig_loads)
diff --git a/pandas/compat/pyarrow.py b/pandas/compat/pyarrow.py
index beb4814914101..5a96e5a4cc49a 100644
--- a/pandas/compat/pyarrow.py
+++ b/pandas/compat/pyarrow.py
@@ -1,4 +1,4 @@
-""" support pyarrow compatibility across versions """
+"""support pyarrow compatibility across versions"""
from __future__ import annotations
@@ -15,6 +15,7 @@
pa_version_under14p0 = _palv < Version("14.0.0")
pa_version_under14p1 = _palv < Version("14.0.1")
pa_version_under15p0 = _palv < Version("15.0.0")
+ pa_version_under16p0 = _palv < Version("16.0.0")
except ImportError:
pa_version_under10p1 = True
pa_version_under11p0 = True
@@ -23,3 +24,4 @@
pa_version_under14p0 = True
pa_version_under14p1 = True
pa_version_under15p0 = True
+ pa_version_under16p0 = True
diff --git a/pandas/conftest.py b/pandas/conftest.py
index 046cda259eefd..34489bb70575a 100644
--- a/pandas/conftest.py
+++ b/pandas/conftest.py
@@ -17,6 +17,7 @@
- Dtypes
- Misc
"""
+
from __future__ import annotations
from collections import abc
@@ -28,12 +29,14 @@
timezone,
)
from decimal import Decimal
+import gc
import operator
import os
from typing import (
TYPE_CHECKING,
Callable,
)
+import uuid
from dateutil.tz import (
tzlocal,
@@ -48,8 +51,6 @@
utc,
)
-from pandas._config.config import _get_option
-
import pandas.util._test_decorators as td
from pandas.core.dtypes.dtypes import (
@@ -124,7 +125,7 @@ def ignore_doctest_warning(item: pytest.Item, path: str, message: str) -> None:
item : pytest.Item
pytest test item.
path : str
- Module path to Python object, e.g. "pandas.core.frame.DataFrame.append". A
+ Module path to Python object, e.g. "pandas.DataFrame.append". A
warning will be filtered when item.name ends with in given path. So it is
sufficient to specify e.g. "DataFrame.append".
message : str
@@ -149,7 +150,6 @@ def pytest_collection_modifyitems(items, config) -> None:
("is_categorical_dtype", "is_categorical_dtype is deprecated"),
("is_sparse", "is_sparse is deprecated"),
("DataFrameGroupBy.fillna", "DataFrameGroupBy.fillna is deprecated"),
- ("NDFrame.replace", "The 'method' keyword"),
("NDFrame.replace", "Series.replace without 'value'"),
("NDFrame.clip", "Downcasting behavior in Series and DataFrame methods"),
("Series.idxmin", "The behavior of Series.idxmin"),
@@ -159,15 +159,6 @@ def pytest_collection_modifyitems(items, config) -> None:
("SeriesGroupBy.idxmax", "The behavior of Series.idxmax"),
# Docstring divides by zero to show behavior difference
("missing.mask_zero_div_zero", "divide by zero encountered"),
- (
- "to_pydatetime",
- "The behavior of DatetimeProperties.to_pydatetime is deprecated",
- ),
- (
- "pandas.core.generic.NDFrame.bool",
- "(Series|DataFrame).bool is now deprecated and will be removed "
- "in future version of pandas",
- ),
(
"pandas.core.generic.NDFrame.first",
"first is deprecated and will be removed in a future version. "
@@ -181,19 +172,11 @@ def pytest_collection_modifyitems(items, config) -> None:
"DataFrameGroupBy.fillna",
"DataFrameGroupBy.fillna with 'method' is deprecated",
),
- (
- "DataFrameGroupBy.fillna",
- "DataFrame.fillna with 'method' is deprecated",
- ),
("read_parquet", "Passing a BlockManager to DataFrame is deprecated"),
]
if is_doctest:
for item in items:
- # autouse=True for the add_doctest_imports can lead to expensive teardowns
- # since doctest_namespace is a session fixture
- item.add_marker(pytest.mark.usefixtures("add_doctest_imports"))
-
for path, message in ignored_doctest_warnings:
ignore_doctest_warning(item, path, message)
@@ -250,7 +233,14 @@ def pytest_collection_modifyitems(items, config) -> None:
)
-@pytest.fixture
+# ----------------------------------------------------------------
+# Autouse fixtures
+# ----------------------------------------------------------------
+
+
+# https://github.com/pytest-dev/pytest/issues/11873
+# Would like to avoid autouse=True, but cannot as of pytest 8.0.0
+@pytest.fixture(autouse=True)
def add_doctest_imports(doctest_namespace) -> None:
"""
Make `np` and `pd` names available for doctests.
@@ -259,9 +249,6 @@ def add_doctest_imports(doctest_namespace) -> None:
doctest_namespace["pd"] = pd
-# ----------------------------------------------------------------
-# Autouse fixtures
-# ----------------------------------------------------------------
@pytest.fixture(autouse=True)
def configure_tests() -> None:
"""
@@ -273,7 +260,7 @@ def configure_tests() -> None:
# ----------------------------------------------------------------
# Common arguments
# ----------------------------------------------------------------
-@pytest.fixture(params=[0, 1, "index", "columns"], ids=lambda x: f"axis={repr(x)}")
+@pytest.fixture(params=[0, 1, "index", "columns"], ids=lambda x: f"axis={x!r}")
def axis(request):
"""
Fixture for returning the axis numbers of a DataFrame.
@@ -281,18 +268,7 @@ def axis(request):
return request.param
-axis_frame = axis
-
-
-@pytest.fixture(params=[1, "columns"], ids=lambda x: f"axis={repr(x)}")
-def axis_1(request):
- """
- Fixture for returning aliases of axis 1 of a DataFrame.
- """
- return request.param
-
-
-@pytest.fixture(params=[True, False, None])
+@pytest.fixture(params=[True, False])
def observed(request):
"""
Pass in the observed keyword to groupby for [True, False]
@@ -313,6 +289,22 @@ def ordered(request):
return request.param
+@pytest.fixture(params=[True, False])
+def dropna(request):
+ """
+ Boolean 'dropna' parameter.
+ """
+ return request.param
+
+
+@pytest.fixture(params=[True, False])
+def sort(request):
+ """
+ Boolean 'sort' parameter.
+ """
+ return request.param
+
+
@pytest.fixture(params=[True, False])
def skipna(request):
"""
@@ -414,6 +406,74 @@ def nselect_method(request):
return request.param
+@pytest.fixture(params=[None, "ignore"])
+def na_action(request):
+ """
+ Fixture for 'na_action' argument in map.
+ """
+ return request.param
+
+
+@pytest.fixture(params=[True, False])
+def ascending(request):
+ """
+ Fixture for 'na_action' argument in sort_values/sort_index/rank.
+ """
+ return request.param
+
+
+@pytest.fixture(params=["average", "min", "max", "first", "dense"])
+def rank_method(request):
+ """
+ Fixture for 'rank' argument in rank.
+ """
+ return request.param
+
+
+@pytest.fixture(params=[True, False])
+def as_index(request):
+ """
+ Fixture for 'as_index' argument in groupby.
+ """
+ return request.param
+
+
+@pytest.fixture(params=[True, False])
+def cache(request):
+ """
+ Fixture for 'cache' argument in to_datetime.
+ """
+ return request.param
+
+
+@pytest.fixture(params=[True, False])
+def parallel(request):
+ """
+ Fixture for parallel keyword argument for numba.jit.
+ """
+ return request.param
+
+
+# Can parameterize nogil & nopython over True | False, but limiting per
+# https://github.com/pandas-dev/pandas/pull/41971#issuecomment-860607472
+
+
+@pytest.fixture(params=[False])
+def nogil(request):
+ """
+ Fixture for nogil keyword argument for numba.jit.
+ """
+ return request.param
+
+
+@pytest.fixture(params=[True])
+def nopython(request):
+ """
+ Fixture for nopython keyword argument for numba.jit.
+ """
+ return request.param
+
+
# ----------------------------------------------------------------
# Missing values & co.
# ----------------------------------------------------------------
@@ -478,10 +538,6 @@ def index_or_series(request):
return request.param
-# Generate cartesian product of index_or_series fixture:
-index_or_series2 = index_or_series
-
-
@pytest.fixture(params=[Index, Series, pd.array], ids=["index", "series", "array"])
def index_or_series_or_array(request):
"""
@@ -674,10 +730,6 @@ def index(request):
return indices_dict[request.param].copy()
-# Needed to generate cartesian product of indices
-index_fixture2 = index
-
-
@pytest.fixture(
params=[
key for key, value in indices_dict.items() if not isinstance(value, MultiIndex)
@@ -691,10 +743,6 @@ def index_flat(request):
return indices_dict[key].copy()
-# Alias so we can test with cartesian product of index_flat
-index_flat2 = index_flat
-
-
@pytest.fixture(
params=[
key
@@ -1182,10 +1230,6 @@ def tz_aware_fixture(request):
return request.param
-# Generate cartesian product of tz_aware_fixture:
-tz_aware_fixture2 = tz_aware_fixture
-
-
_UTCS = ["utc", "dateutil/UTC", utc, tzutc(), timezone.utc]
if zoneinfo is not None:
_UTCS.append(zoneinfo.ZoneInfo("UTC"))
@@ -1349,7 +1393,7 @@ def fixed_now_ts() -> Timestamp:
"""
Fixture emits fixed Timestamp.now()
"""
- return Timestamp( # pyright: ignore[reportGeneralTypeIssues]
+ return Timestamp( # pyright: ignore[reportReturnType]
year=2021, month=1, day=1, hour=12, minute=4, second=13, microsecond=22
)
@@ -1642,6 +1686,38 @@ def any_numpy_dtype(request):
return request.param
+@pytest.fixture(params=tm.ALL_REAL_NULLABLE_DTYPES)
+def any_real_nullable_dtype(request):
+ """
+ Parameterized fixture for all real dtypes that can hold NA.
+
+ * float
+ * 'float32'
+ * 'float64'
+ * 'Float32'
+ * 'Float64'
+ * 'UInt8'
+ * 'UInt16'
+ * 'UInt32'
+ * 'UInt64'
+ * 'Int8'
+ * 'Int16'
+ * 'Int32'
+ * 'Int64'
+ * 'uint8[pyarrow]'
+ * 'uint16[pyarrow]'
+ * 'uint32[pyarrow]'
+ * 'uint64[pyarrow]'
+ * 'int8[pyarrow]'
+ * 'int16[pyarrow]'
+ * 'int32[pyarrow]'
+ * 'int64[pyarrow]'
+ * 'float[pyarrow]'
+ * 'double[pyarrow]'
+ """
+ return request.param
+
+
@pytest.fixture(params=tm.ALL_NUMERIC_DTYPES)
def any_numeric_dtype(request):
"""
@@ -1777,14 +1853,37 @@ def ip():
return InteractiveShell(config=c)
-@pytest.fixture(params=["bsr", "coo", "csc", "csr", "dia", "dok", "lil"])
-def spmatrix(request):
- """
- Yields scipy sparse matrix classes.
+@pytest.fixture
+def mpl_cleanup():
"""
- sparse = pytest.importorskip("scipy.sparse")
+ Ensure Matplotlib is cleaned up around a test.
+
+ Before a test is run:
+
+ 1) Set the backend to "template" to avoid requiring a GUI.
+
+ After a test is run:
+
+ 1) Reset units registry
+ 2) Reset rc_context
+ 3) Close all figures
- return getattr(sparse, request.param + "_matrix")
+ See matplotlib/testing/decorators.py#L24.
+ """
+ mpl = pytest.importorskip("matplotlib")
+ mpl_units = pytest.importorskip("matplotlib.units")
+ plt = pytest.importorskip("matplotlib.pyplot")
+ orig_units_registry = mpl_units.registry.copy()
+ try:
+ with mpl.rc_context():
+ mpl.use("template")
+ yield
+ finally:
+ mpl_units.registry.clear()
+ mpl_units.registry.update(orig_units_registry)
+ plt.close("all")
+ # https://matplotlib.org/stable/users/prev_whats_new/whats_new_3.6.0.html#garbage-collection-is-no-longer-run-on-figure-close # noqa: E501
+ gc.collect(1)
@pytest.fixture(
@@ -1877,26 +1976,14 @@ def indexer_ial(request):
return request.param
-@pytest.fixture
-def using_copy_on_write() -> bool:
- """
- Fixture to check if Copy-on-Write is enabled.
- """
- return (
- pd.options.mode.copy_on_write is True
- and _get_option("mode.data_manager", silent=True) == "block"
- )
-
-
-@pytest.fixture
-def warn_copy_on_write() -> bool:
+@pytest.fixture(params=[True, False])
+def performance_warning(request) -> Iterator[bool | type[Warning]]:
"""
- Fixture to check if Copy-on-Write is in warning mode.
+ Fixture to check if performance warnings are enabled. Either produces
+ ``PerformanceWarning`` if they are enabled, otherwise ``False``.
"""
- return (
- pd.options.mode.copy_on_write == "warn"
- and _get_option("mode.data_manager", silent=True) == "block"
- )
+ with pd.option_context("mode.performance_warnings", request.param):
+ yield pd.errors.PerformanceWarning if request.param else False
@pytest.fixture
@@ -1920,6 +2007,20 @@ def warsaw(request) -> str:
return request.param
-@pytest.fixture()
+@pytest.fixture
def arrow_string_storage():
+ """
+ Fixture that lists possible PyArrow values for StringDtype storage field.
+ """
return ("pyarrow", "pyarrow_numpy")
+
+
+@pytest.fixture
+def temp_file(tmp_path):
+ """
+ Generate a unique file for testing use. See link for removal policy.
+ https://docs.pytest.org/en/7.1.x/how-to/tmp_path.html#the-default-base-temporary-directory
+ """
+ file_path = tmp_path / str(uuid.uuid4())
+ file_path.touch()
+ return file_path
diff --git a/pandas/core/_numba/extensions.py b/pandas/core/_numba/extensions.py
index ee09c9380fb0f..e6f0427de2a3a 100644
--- a/pandas/core/_numba/extensions.py
+++ b/pandas/core/_numba/extensions.py
@@ -12,6 +12,7 @@
from contextlib import contextmanager
import operator
+from typing import TYPE_CHECKING
import numba
from numba import types
@@ -40,6 +41,9 @@
from pandas.core.internals import SingleBlockManager
from pandas.core.series import Series
+if TYPE_CHECKING:
+ from pandas._typing import Self
+
# Helper function to hack around fact that Index casts numpy string dtype to object
#
@@ -84,7 +88,7 @@ def key(self):
def as_array(self):
return types.Array(self.dtype, 1, self.layout)
- def copy(self, dtype=None, ndim: int = 1, layout=None):
+ def copy(self, dtype=None, ndim: int = 1, layout=None) -> Self:
assert ndim == 1
if dtype is None:
dtype = self.dtype
@@ -114,7 +118,7 @@ def key(self):
def as_array(self):
return self.values
- def copy(self, dtype=None, ndim: int = 1, layout: str = "C"):
+ def copy(self, dtype=None, ndim: int = 1, layout: str = "C") -> Self:
assert ndim == 1
assert layout == "C"
if dtype is None:
@@ -123,7 +127,7 @@ def copy(self, dtype=None, ndim: int = 1, layout: str = "C"):
@typeof_impl.register(Index)
-def typeof_index(val, c):
+def typeof_index(val, c) -> IndexType:
"""
This will assume that only strings are in object dtype
index.
@@ -136,7 +140,7 @@ def typeof_index(val, c):
@typeof_impl.register(Series)
-def typeof_series(val, c):
+def typeof_series(val, c) -> SeriesType:
index = typeof_impl(val.index, c)
arrty = typeof_impl(val.values, c)
namety = typeof_impl(val.name, c)
@@ -532,7 +536,7 @@ def key(self):
@typeof_impl.register(_iLocIndexer)
-def typeof_iloc(val, c):
+def typeof_iloc(val, c) -> IlocType:
objtype = typeof_impl(val.obj, c)
return IlocType(objtype)
diff --git a/pandas/core/_numba/kernels/mean_.py b/pandas/core/_numba/kernels/mean_.py
index f415804781753..cc10bd003af7e 100644
--- a/pandas/core/_numba/kernels/mean_.py
+++ b/pandas/core/_numba/kernels/mean_.py
@@ -6,6 +6,7 @@
Mirrors pandas/_libs/window/aggregation.pyx
"""
+
from __future__ import annotations
from typing import TYPE_CHECKING
@@ -107,7 +108,7 @@ def sliding_mean(
neg_ct,
compensation_add,
num_consecutive_same_value,
- prev_value, # pyright: ignore[reportGeneralTypeIssues]
+ prev_value, # pyright: ignore[reportArgumentType]
)
else:
for j in range(start[i - 1], s):
@@ -132,7 +133,7 @@ def sliding_mean(
neg_ct,
compensation_add,
num_consecutive_same_value,
- prev_value, # pyright: ignore[reportGeneralTypeIssues]
+ prev_value, # pyright: ignore[reportArgumentType]
)
if nobs >= min_periods and nobs > 0:
diff --git a/pandas/core/_numba/kernels/min_max_.py b/pandas/core/_numba/kernels/min_max_.py
index c9803980e64a6..59d36732ebae6 100644
--- a/pandas/core/_numba/kernels/min_max_.py
+++ b/pandas/core/_numba/kernels/min_max_.py
@@ -6,6 +6,7 @@
Mirrors pandas/_libs/window/aggregation.pyx
"""
+
from __future__ import annotations
from typing import TYPE_CHECKING
diff --git a/pandas/core/_numba/kernels/sum_.py b/pandas/core/_numba/kernels/sum_.py
index 94db84267ceec..76f4e22b43c4b 100644
--- a/pandas/core/_numba/kernels/sum_.py
+++ b/pandas/core/_numba/kernels/sum_.py
@@ -6,6 +6,7 @@
Mirrors pandas/_libs/window/aggregation.pyx
"""
+
from __future__ import annotations
from typing import (
diff --git a/pandas/core/_numba/kernels/var_.py b/pandas/core/_numba/kernels/var_.py
index c63d0b90b0fc3..69aec4d6522c4 100644
--- a/pandas/core/_numba/kernels/var_.py
+++ b/pandas/core/_numba/kernels/var_.py
@@ -6,6 +6,7 @@
Mirrors pandas/_libs/window/aggregation.pyx
"""
+
from __future__ import annotations
from typing import TYPE_CHECKING
diff --git a/pandas/core/accessor.py b/pandas/core/accessor.py
index 698abb2ec4989..99b5053ce250c 100644
--- a/pandas/core/accessor.py
+++ b/pandas/core/accessor.py
@@ -4,6 +4,7 @@
that can be mixed into or pinned onto other pandas classes.
"""
+
from __future__ import annotations
from typing import (
@@ -54,7 +55,7 @@ class PandasDelegate:
def _delegate_property_get(self, name: str, *args, **kwargs):
raise TypeError(f"You cannot access the property {name}")
- def _delegate_property_set(self, name: str, value, *args, **kwargs):
+ def _delegate_property_set(self, name: str, value, *args, **kwargs) -> None:
raise TypeError(f"The property {name} cannot be set")
def _delegate_method(self, name: str, *args, **kwargs):
@@ -265,7 +266,7 @@ def __init__(self, pandas_object): # noqa: E999
For consistency with pandas methods, you should raise an ``AttributeError``
if the data passed to your accessor has an incorrect dtype.
- >>> pd.Series(['a', 'b']).dt
+ >>> pd.Series(["a", "b"]).dt
Traceback (most recent call last):
...
AttributeError: Can only use .dt accessor with datetimelike values
@@ -274,8 +275,6 @@ def __init__(self, pandas_object): # noqa: E999
--------
In your library code::
- import pandas as pd
-
@pd.api.extensions.register_dataframe_accessor("geo")
class GeoAccessor:
def __init__(self, pandas_obj):
@@ -306,8 +305,8 @@ def plot(self):
def decorator(accessor):
if hasattr(cls, name):
warnings.warn(
- f"registration of accessor {repr(accessor)} under name "
- f"{repr(name)} for type {repr(cls)} is overriding a preexisting "
+ f"registration of accessor {accessor!r} under name "
+ f"{name!r} for type {cls!r} is overriding a preexisting "
f"attribute with the same name.",
UserWarning,
stacklevel=find_stack_level(),
diff --git a/pandas/core/algorithms.py b/pandas/core/algorithms.py
index 15a07da76d2f7..33beef23197bd 100644
--- a/pandas/core/algorithms.py
+++ b/pandas/core/algorithms.py
@@ -2,6 +2,7 @@
Generic data algorithms. This module is experimental at the moment and not
intended for public consumption
"""
+
from __future__ import annotations
import decimal
@@ -25,6 +26,7 @@
from pandas._typing import (
AnyArrayLike,
ArrayLike,
+ ArrayLikeT,
AxisInt,
DtypeObj,
TakeIndexer,
@@ -41,11 +43,11 @@
ensure_float64,
ensure_object,
ensure_platform_int,
- is_array_like,
is_bool_dtype,
is_complex_dtype,
is_dict_like,
is_extension_array_dtype,
+ is_float,
is_float_dtype,
is_integer,
is_integer_dtype,
@@ -182,8 +184,8 @@ def _ensure_data(values: ArrayLike) -> np.ndarray:
def _reconstruct_data(
- values: ArrayLike, dtype: DtypeObj, original: AnyArrayLike
-) -> ArrayLike:
+ values: ArrayLikeT, dtype: DtypeObj, original: AnyArrayLike
+) -> ArrayLikeT:
"""
reverse of _ensure_data
@@ -206,7 +208,9 @@ def _reconstruct_data(
# that values.dtype == dtype
cls = dtype.construct_array_type()
- values = cls._from_sequence(values, dtype=dtype)
+ # error: Incompatible types in assignment (expression has type
+ # "ExtensionArray", variable has type "ndarray[Any, Any]")
+ values = cls._from_sequence(values, dtype=dtype) # type: ignore[assignment]
else:
values = values.astype(dtype, copy=False)
@@ -222,12 +226,9 @@ def _ensure_arraylike(values, func_name: str) -> ArrayLike:
# GH#52986
if func_name != "isin-targets":
# Make an exception for the comps argument in isin.
- warnings.warn(
- f"{func_name} with argument that is not not a Series, Index, "
- "ExtensionArray, or np.ndarray is deprecated and will raise in a "
- "future version.",
- FutureWarning,
- stacklevel=find_stack_level(),
+ raise TypeError(
+ f"{func_name} requires a Series, Index, "
+ f"ExtensionArray, or np.ndarray, got {type(values).__name__}."
)
inferred = lib.infer_dtype(values, skipna=False)
@@ -259,7 +260,9 @@ def _ensure_arraylike(values, func_name: str) -> ArrayLike:
}
-def _get_hashtable_algo(values: np.ndarray):
+def _get_hashtable_algo(
+ values: np.ndarray,
+) -> tuple[type[htable.HashTable], np.ndarray]:
"""
Parameters
----------
@@ -432,6 +435,10 @@ def unique_with_mask(values, mask: npt.NDArray[np.bool_] | None = None):
# Dispatch to extension dtype's unique.
return values.unique()
+ if isinstance(values, ABCIndex):
+ # Dispatch to Index's unique.
+ return values.unique()
+
original = values
hashtable, values = _get_hashtable_algo(values)
@@ -812,53 +819,6 @@ def factorize(
return codes, uniques
-def value_counts(
- values,
- sort: bool = True,
- ascending: bool = False,
- normalize: bool = False,
- bins=None,
- dropna: bool = True,
-) -> Series:
- """
- Compute a histogram of the counts of non-null values.
-
- Parameters
- ----------
- values : ndarray (1-d)
- sort : bool, default True
- Sort by values
- ascending : bool, default False
- Sort in ascending order
- normalize: bool, default False
- If True then compute a relative histogram
- bins : integer, optional
- Rather than count values, group them into half-open bins,
- convenience for pd.cut, only works with numeric data
- dropna : bool, default True
- Don't include counts of NaN
-
- Returns
- -------
- Series
- """
- warnings.warn(
- # GH#53493
- "pandas.value_counts is deprecated and will be removed in a "
- "future version. Use pd.Series(obj).value_counts() instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return value_counts_internal(
- values,
- sort=sort,
- ascending=ascending,
- normalize=normalize,
- bins=bins,
- dropna=dropna,
- )
-
-
def value_counts_internal(
values,
sort: bool = True,
@@ -928,26 +888,9 @@ def value_counts_internal(
if keys.dtype == np.float16:
keys = keys.astype(np.float32)
- # For backwards compatibility, we let Index do its normal type
- # inference, _except_ for if if infers from object to bool.
- idx = Index(keys)
- if idx.dtype == bool and keys.dtype == object:
- idx = idx.astype(object)
- elif (
- idx.dtype != keys.dtype # noqa: PLR1714 # # pylint: disable=R1714
- and idx.dtype != "string[pyarrow_numpy]"
- ):
- warnings.warn(
- # GH#56161
- "The behavior of value_counts with object-dtype is deprecated. "
- "In a future version, this will *not* perform dtype inference "
- "on the resulting index. To retain the old behavior, use "
- "`result.index = result.index.infer_objects()`",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- idx.name = index_name
-
+ # Starting in 3.0, we no longer perform dtype inference on the
+ # Index object we construct here, xref GH#56161
+ idx = Index(keys, dtype=keys.dtype, name=index_name)
result = Series(counts, index=idx, name=name, copy=False)
if sort:
@@ -1209,34 +1152,37 @@ def take(
>>> pd.api.extensions.take(np.array([10, 20, 30]), [0, 0, -1], allow_fill=True)
array([10., 10., nan])
- >>> pd.api.extensions.take(np.array([10, 20, 30]), [0, 0, -1], allow_fill=True,
- ... fill_value=-10)
+ >>> pd.api.extensions.take(
+ ... np.array([10, 20, 30]), [0, 0, -1], allow_fill=True, fill_value=-10
+ ... )
array([ 10, 10, -10])
"""
if not isinstance(arr, (np.ndarray, ABCExtensionArray, ABCIndex, ABCSeries)):
# GH#52981
- warnings.warn(
- "pd.api.extensions.take accepting non-standard inputs is deprecated "
- "and will raise in a future version. Pass either a numpy.ndarray, "
- "ExtensionArray, Index, or Series instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
+ raise TypeError(
+ "pd.api.extensions.take requires a numpy.ndarray, "
+ f"ExtensionArray, Index, or Series, got {type(arr).__name__}."
)
- if not is_array_like(arr):
- arr = np.asarray(arr)
-
indices = ensure_platform_int(indices)
if allow_fill:
# Pandas style, -1 means NA
validate_indices(indices, arr.shape[axis])
+ # error: Argument 1 to "take_nd" has incompatible type
+ # "ndarray[Any, Any] | ExtensionArray | Index | Series"; expected
+ # "ndarray[Any, Any]"
result = take_nd(
- arr, indices, axis=axis, allow_fill=True, fill_value=fill_value
+ arr, # type: ignore[arg-type]
+ indices,
+ axis=axis,
+ allow_fill=True,
+ fill_value=fill_value,
)
else:
# NumPy style
- result = arr.take(indices, axis=axis)
+ # error: Unexpected keyword argument "axis" for "take" of "ExtensionArray"
+ result = arr.take(indices, axis=axis) # type: ignore[call-arg,assignment]
return result
@@ -1356,7 +1302,12 @@ def diff(arr, n: int, axis: AxisInt = 0):
shifted
"""
- n = int(n)
+ # added a check on the integer value of period
+ # see https://github.com/pandas-dev/pandas/issues/56607
+ if not lib.is_integer(n):
+ if not (is_float(n) and n.is_integer()):
+ raise ValueError("periods must be an integer")
+ n = int(n)
na = np.nan
dtype = arr.dtype
@@ -1550,7 +1501,9 @@ def safe_sort(
hash_klass, values = _get_hashtable_algo(values) # type: ignore[arg-type]
t = hash_klass(len(values))
t.map_locations(values)
- sorter = ensure_platform_int(t.lookup(ordered))
+ # error: Argument 1 to "lookup" of "HashTable" has incompatible type
+ # "ExtensionArray | ndarray[Any, Any] | Index | Series"; expected "ndarray"
+ sorter = ensure_platform_int(t.lookup(ordered)) # type: ignore[arg-type]
if use_na_sentinel:
# take_nd is faster, but only works for na_sentinels of -1
@@ -1634,16 +1587,8 @@ def union_with_duplicates(
"""
from pandas import Series
- with warnings.catch_warnings():
- # filter warning from object dtype inference; we will end up discarding
- # the index here, so the deprecation does not affect the end result here.
- warnings.filterwarnings(
- "ignore",
- "The behavior of value_counts with object-dtype is deprecated",
- category=FutureWarning,
- )
- l_count = value_counts_internal(lvals, dropna=False)
- r_count = value_counts_internal(rvals, dropna=False)
+ l_count = value_counts_internal(lvals, dropna=False)
+ r_count = value_counts_internal(rvals, dropna=False)
l_count, r_count = l_count.align(r_count, fill_value=0)
final_count = np.maximum(l_count.values, r_count.values)
final_count = Series(final_count, index=l_count.index, dtype="int", copy=False)
@@ -1668,7 +1613,6 @@ def map_array(
arr: ArrayLike,
mapper,
na_action: Literal["ignore"] | None = None,
- convert: bool = True,
) -> np.ndarray | ExtensionArray | Index:
"""
Map values using an input mapping or function.
@@ -1680,9 +1624,6 @@ def map_array(
na_action : {None, 'ignore'}, default None
If 'ignore', propagate NA values, without passing them to the
mapping correspondence.
- convert : bool, default True
- Try to find better dtype for elementwise function results. If
- False, leave as dtype=object.
Returns
-------
@@ -1740,8 +1681,6 @@ def map_array(
# we must convert to python types
values = arr.astype(object, copy=False)
if na_action is None:
- return lib.map_infer(values, mapper, convert=convert)
+ return lib.map_infer(values, mapper)
else:
- return lib.map_infer_mask(
- values, mapper, mask=isna(values).view(np.uint8), convert=convert
- )
+ return lib.map_infer_mask(values, mapper, mask=isna(values).view(np.uint8))
diff --git a/pandas/core/api.py b/pandas/core/api.py
index 2cfe5ffc0170d..3d2e855831c05 100644
--- a/pandas/core/api.py
+++ b/pandas/core/api.py
@@ -23,7 +23,6 @@
from pandas.core.algorithms import (
factorize,
unique,
- value_counts,
)
from pandas.core.arrays import Categorical
from pandas.core.arrays.boolean import BooleanDtype
@@ -136,5 +135,4 @@
"UInt64Dtype",
"UInt8Dtype",
"unique",
- "value_counts",
]
diff --git a/pandas/core/apply.py b/pandas/core/apply.py
index 784e11415ade6..e8df24850f7a8 100644
--- a/pandas/core/apply.py
+++ b/pandas/core/apply.py
@@ -12,13 +12,9 @@
Literal,
cast,
)
-import warnings
import numpy as np
-from pandas._config import option_context
-
-from pandas._libs import lib
from pandas._libs.internals import BlockValuesRefs
from pandas._typing import (
AggFuncType,
@@ -33,7 +29,6 @@
from pandas.compat._optional import import_optional_dependency
from pandas.errors import SpecificationError
from pandas.util._decorators import cache_readonly
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.cast import is_nested_object
from pandas.core.dtypes.common import (
@@ -178,10 +173,7 @@ def agg(self) -> DataFrame | Series | None:
Result of aggregation, or None if agg cannot be performed by
this method.
"""
- obj = self.obj
func = self.func
- args = self.args
- kwargs = self.kwargs
if isinstance(func, str):
return self.apply_str()
@@ -192,12 +184,6 @@ def agg(self) -> DataFrame | Series | None:
# we require a list, but not a 'str'
return self.agg_list_like()
- if callable(func):
- f = com.get_cython_func(func)
- if f and not args and not kwargs:
- warn_alias_replacement(obj, func, f)
- return getattr(obj, f)()
-
# caller can react
return None
@@ -303,12 +289,6 @@ def transform_str_or_callable(self, func) -> DataFrame | Series:
if isinstance(func, str):
return self._apply_str(obj, func, *args, **kwargs)
- if not args and not kwargs:
- f = com.get_cython_func(func)
- if f:
- warn_alias_replacement(obj, func, f)
- return getattr(obj, f)()
-
# Two possible ways to use a UDF - apply or call directly
try:
return obj.apply(func, args=args, **kwargs)
@@ -584,22 +564,10 @@ def apply_str(self) -> DataFrame | Series:
"axis" not in arg_names or func in ("corrwith", "skew")
):
raise ValueError(f"Operation {func} does not support axis=1")
- if "axis" in arg_names:
- if isinstance(obj, (SeriesGroupBy, DataFrameGroupBy)):
- # Try to avoid FutureWarning for deprecated axis keyword;
- # If self.axis matches the axis we would get by not passing
- # axis, we safely exclude the keyword.
-
- default_axis = 0
- if func in ["idxmax", "idxmin"]:
- # DataFrameGroupBy.idxmax, idxmin axis defaults to self.axis,
- # whereas other axis keywords default to 0
- default_axis = self.obj.axis
-
- if default_axis != self.axis:
- self.kwargs["axis"] = self.axis
- else:
- self.kwargs["axis"] = self.axis
+ if "axis" in arg_names and not isinstance(
+ obj, (SeriesGroupBy, DataFrameGroupBy)
+ ):
+ self.kwargs["axis"] = self.axis
return self._apply_str(obj, func, *self.args, **self.kwargs)
def apply_list_or_dict_like(self) -> DataFrame | Series:
@@ -1010,7 +978,8 @@ def wrapper(*args, **kwargs):
# [..., Any] | str] | dict[Hashable,Callable[..., Any] | str |
# list[Callable[..., Any] | str]]"; expected "Hashable"
nb_looper = generate_apply_looper(
- self.func, **engine_kwargs # type: ignore[arg-type]
+ self.func, # type: ignore[arg-type]
+ **engine_kwargs,
)
result = nb_looper(self.values, self.axis)
# If we made the result 2-D, squeeze it back to 1-D
@@ -1075,14 +1044,12 @@ def apply_series_generator(self) -> tuple[ResType, Index]:
results = {}
- with option_context("mode.chained_assignment", None):
- for i, v in enumerate(series_gen):
- # ignore SettingWithCopy here in case the user mutates
- results[i] = self.func(v, *self.args, **self.kwargs)
- if isinstance(results[i], ABCSeries):
- # If we have a view on v, we need to make a copy because
- # series_generator will swap out the underlying data
- results[i] = results[i].copy(deep=False)
+ for i, v in enumerate(series_gen):
+ results[i] = self.func(v, *self.args, **self.kwargs)
+ if isinstance(results[i], ABCSeries):
+ # If we have a view on v, we need to make a copy because
+ # series_generator will swap out the underlying data
+ results[i] = results[i].copy(deep=False)
return results, res_index
@@ -1255,7 +1222,7 @@ def series_generator(self) -> Generator[Series, None, None]:
ser = self.obj._ixs(0, axis=0)
mgr = ser._mgr
- is_view = mgr.blocks[0].refs.has_reference() # type: ignore[union-attr]
+ is_view = mgr.blocks[0].refs.has_reference()
if isinstance(ser.dtype, ExtensionDtype):
# values will be incorrect for this block
@@ -1277,7 +1244,7 @@ def series_generator(self) -> Generator[Series, None, None]:
# -> if that happened and `ser` is already a copy, then we reset
# the refs here to avoid triggering a unnecessary CoW inside the
# applied function (https://github.com/pandas-dev/pandas/pull/56212)
- mgr.blocks[0].refs = BlockValuesRefs(mgr.blocks[0]) # type: ignore[union-attr]
+ mgr.blocks[0].refs = BlockValuesRefs(mgr.blocks[0])
yield ser
@staticmethod
@@ -1319,9 +1286,10 @@ def apply_with_numba(self) -> dict[int, Any]:
# Convert from numba dict to regular dict
# Our isinstance checks in the df constructor don't pass for numbas typed dict
- with set_numba_data(self.obj.index) as index, set_numba_data(
- self.columns
- ) as columns:
+ with (
+ set_numba_data(self.obj.index) as index,
+ set_numba_data(self.columns) as columns,
+ ):
res = dict(nb_func(self.values, columns, index))
return res
@@ -1364,7 +1332,7 @@ def infer_to_same_shape(self, results: ResType, res_index: Index) -> DataFrame:
result.index = res_index
# infer dtypes
- result = result.infer_objects(copy=False)
+ result = result.infer_objects()
return result
@@ -1379,23 +1347,10 @@ def __init__(
obj: Series,
func: AggFuncType,
*,
- convert_dtype: bool | lib.NoDefault = lib.no_default,
by_row: Literal[False, "compat", "_compat"] = "compat",
args,
kwargs,
) -> None:
- if convert_dtype is lib.no_default:
- convert_dtype = True
- else:
- warnings.warn(
- "the convert_dtype parameter is deprecated and will be removed in a "
- "future version. Do ``ser.astype(object).apply()`` "
- "instead if you want ``convert_dtype=False``.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- self.convert_dtype = convert_dtype
-
super().__init__(
obj,
func,
@@ -1433,22 +1388,7 @@ def agg(self):
func = self.func
# string, list-like, and dict-like are entirely handled in super
assert callable(func)
-
- # GH53325: The setup below is just to keep current behavior while emitting a
- # deprecation message. In the future this will all be replaced with a simple
- # `result = f(self.obj, *self.args, **self.kwargs)`.
- try:
- result = obj.apply(func, args=self.args, **self.kwargs)
- except (ValueError, AttributeError, TypeError):
- result = func(obj, *self.args, **self.kwargs)
- else:
- msg = (
- f"using {func} in {type(obj).__name__}.agg cannot aggregate and "
- f"has been deprecated. Use {type(obj).__name__}.transform to "
- f"keep behavior unchanged."
- )
- warnings.warn(msg, FutureWarning, stacklevel=find_stack_level())
-
+ result = func(obj, *self.args, **self.kwargs)
return result
def apply_empty_result(self) -> Series:
@@ -1504,9 +1444,7 @@ def curried(x):
# TODO: remove the `na_action="ignore"` when that default has been changed in
# Categorical (GH51645).
action = "ignore" if isinstance(obj.dtype, CategoricalDtype) else None
- mapped = obj._map_values(
- mapper=curried, na_action=action, convert=self.convert_dtype
- )
+ mapped = obj._map_values(mapper=curried, na_action=action)
if len(mapped) and isinstance(mapped[0], ABCSeries):
# GH#43986 Need to do list(mapped) in order to get treated as nested
@@ -1797,14 +1735,14 @@ def normalize_keyword_aggregation(
def _make_unique_kwarg_list(
- seq: Sequence[tuple[Any, Any]]
+ seq: Sequence[tuple[Any, Any]],
) -> Sequence[tuple[Any, Any]]:
"""
Uniquify aggfunc name of the pairs in the order list
Examples:
--------
- >>> kwarg_list = [('a', ''), ('a', ''), ('b', '')]
+ >>> kwarg_list = [("a", ""), ("a", ""), ("b", "")]
>>> _make_unique_kwarg_list(kwarg_list)
[('a', '_0'), ('a', '_1'), ('b', '')]
"""
@@ -1836,7 +1774,7 @@ def relabel_result(
>>> from pandas.core.apply import relabel_result
>>> result = pd.DataFrame(
... {"A": [np.nan, 2, np.nan], "C": [6, np.nan, np.nan], "B": [np.nan, 4, 2.5]},
- ... index=["max", "mean", "min"]
+ ... index=["max", "mean", "min"],
... )
>>> funcs = {"A": ["max"], "C": ["max"], "B": ["mean", "min"]}
>>> columns = ("foo", "aab", "bar", "dat")
@@ -1891,7 +1829,7 @@ def relabel_result(
# assign the new user-provided "named aggregation" as index names, and reindex
# it based on the whole user-provided names.
s.index = reordered_indexes[idx : idx + len(fun)]
- reordered_result_in_dict[col] = s.reindex(columns, copy=False)
+ reordered_result_in_dict[col] = s.reindex(columns)
idx = idx + len(fun)
return reordered_result_in_dict
@@ -1975,7 +1913,7 @@ def maybe_mangle_lambdas(agg_spec: Any) -> Any:
Examples
--------
- >>> maybe_mangle_lambdas('sum')
+ >>> maybe_mangle_lambdas("sum")
'sum'
>>> maybe_mangle_lambdas([lambda: 1, lambda: 2]) # doctest: +SKIP
[,
@@ -2020,7 +1958,7 @@ def validate_func_kwargs(
Examples
--------
- >>> validate_func_kwargs({'one': 'min', 'two': 'max'})
+ >>> validate_func_kwargs({"one": "min", "two": "max"})
(['one', 'two'], ['min', 'max'])
"""
tuple_given_message = "func is expected but received {} in **kwargs."
@@ -2040,23 +1978,3 @@ def include_axis(op_name: Literal["agg", "apply"], colg: Series | DataFrame) ->
return isinstance(colg, ABCDataFrame) or (
isinstance(colg, ABCSeries) and op_name == "agg"
)
-
-
-def warn_alias_replacement(
- obj: AggObjType,
- func: Callable,
- alias: str,
-) -> None:
- if alias.startswith("np."):
- full_alias = alias
- else:
- full_alias = f"{type(obj).__name__}.{alias}"
- alias = f'"{alias}"'
- warnings.warn(
- f"The provided callable {func} is currently using "
- f"{full_alias}. In a future version of pandas, "
- f"the provided callable will be used directly. To keep current "
- f"behavior pass the string {alias} instead.",
- category=FutureWarning,
- stacklevel=find_stack_level(),
- )
diff --git a/pandas/core/array_algos/datetimelike_accumulations.py b/pandas/core/array_algos/datetimelike_accumulations.py
index 825fe60ee6cf8..c3a7c2e4fefb2 100644
--- a/pandas/core/array_algos/datetimelike_accumulations.py
+++ b/pandas/core/array_algos/datetimelike_accumulations.py
@@ -18,7 +18,7 @@ def _cum_func(
values: np.ndarray,
*,
skipna: bool = True,
-):
+) -> np.ndarray:
"""
Accumulations for 1D datetimelike arrays.
@@ -37,8 +37,10 @@ def _cum_func(
np.cumsum: 0,
np.minimum.accumulate: np.iinfo(np.int64).max,
}[func]
- except KeyError:
- raise ValueError(f"No accumulation for {func} implemented on BaseMaskedArray")
+ except KeyError as err:
+ raise ValueError(
+ f"No accumulation for {func} implemented on BaseMaskedArray"
+ ) from err
mask = isna(values)
y = values.view("i8")
@@ -47,7 +49,8 @@ def _cum_func(
if not skipna:
mask = np.maximum.accumulate(mask)
- result = func(y)
+ # GH 57956
+ result = func(y, axis=0)
result[mask] = iNaT
if values.dtype.kind in "mM":
@@ -59,9 +62,9 @@ def cumsum(values: np.ndarray, *, skipna: bool = True) -> np.ndarray:
return _cum_func(np.cumsum, values, skipna=skipna)
-def cummin(values: np.ndarray, *, skipna: bool = True):
+def cummin(values: np.ndarray, *, skipna: bool = True) -> np.ndarray:
return _cum_func(np.minimum.accumulate, values, skipna=skipna)
-def cummax(values: np.ndarray, *, skipna: bool = True):
+def cummax(values: np.ndarray, *, skipna: bool = True) -> np.ndarray:
return _cum_func(np.maximum.accumulate, values, skipna=skipna)
diff --git a/pandas/core/array_algos/masked_accumulations.py b/pandas/core/array_algos/masked_accumulations.py
index ad9e96d398a24..b31d32a606eed 100644
--- a/pandas/core/array_algos/masked_accumulations.py
+++ b/pandas/core/array_algos/masked_accumulations.py
@@ -22,7 +22,7 @@ def _cum_func(
mask: npt.NDArray[np.bool_],
*,
skipna: bool = True,
-):
+) -> tuple[np.ndarray, npt.NDArray[np.bool_]]:
"""
Accumulations for 1D masked array.
@@ -60,10 +60,10 @@ def _cum_func(
np.cumsum: 0,
np.minimum.accumulate: dtype_info.max,
}[func]
- except KeyError:
+ except KeyError as err:
raise NotImplementedError(
f"No accumulation for {func} implemented on BaseMaskedArray"
- )
+ ) from err
values[mask] = fill_value
@@ -74,17 +74,25 @@ def _cum_func(
return values, mask
-def cumsum(values: np.ndarray, mask: npt.NDArray[np.bool_], *, skipna: bool = True):
+def cumsum(
+ values: np.ndarray, mask: npt.NDArray[np.bool_], *, skipna: bool = True
+) -> tuple[np.ndarray, npt.NDArray[np.bool_]]:
return _cum_func(np.cumsum, values, mask, skipna=skipna)
-def cumprod(values: np.ndarray, mask: npt.NDArray[np.bool_], *, skipna: bool = True):
+def cumprod(
+ values: np.ndarray, mask: npt.NDArray[np.bool_], *, skipna: bool = True
+) -> tuple[np.ndarray, npt.NDArray[np.bool_]]:
return _cum_func(np.cumprod, values, mask, skipna=skipna)
-def cummin(values: np.ndarray, mask: npt.NDArray[np.bool_], *, skipna: bool = True):
+def cummin(
+ values: np.ndarray, mask: npt.NDArray[np.bool_], *, skipna: bool = True
+) -> tuple[np.ndarray, npt.NDArray[np.bool_]]:
return _cum_func(np.minimum.accumulate, values, mask, skipna=skipna)
-def cummax(values: np.ndarray, mask: npt.NDArray[np.bool_], *, skipna: bool = True):
+def cummax(
+ values: np.ndarray, mask: npt.NDArray[np.bool_], *, skipna: bool = True
+) -> tuple[np.ndarray, npt.NDArray[np.bool_]]:
return _cum_func(np.maximum.accumulate, values, mask, skipna=skipna)
diff --git a/pandas/core/array_algos/masked_reductions.py b/pandas/core/array_algos/masked_reductions.py
index 335fa1afc0f4e..3784689995802 100644
--- a/pandas/core/array_algos/masked_reductions.py
+++ b/pandas/core/array_algos/masked_reductions.py
@@ -2,6 +2,7 @@
masked_reductions.py is for reduction algorithms using a mask-based approach
for missing values.
"""
+
from __future__ import annotations
from typing import (
diff --git a/pandas/core/array_algos/putmask.py b/pandas/core/array_algos/putmask.py
index f65d2d20e028e..464a4d552af68 100644
--- a/pandas/core/array_algos/putmask.py
+++ b/pandas/core/array_algos/putmask.py
@@ -1,6 +1,7 @@
"""
EA-compatible analogue to np.putmask
"""
+
from __future__ import annotations
from typing import (
diff --git a/pandas/core/array_algos/quantile.py b/pandas/core/array_algos/quantile.py
index ee6f00b219a15..5c933294fb944 100644
--- a/pandas/core/array_algos/quantile.py
+++ b/pandas/core/array_algos/quantile.py
@@ -102,7 +102,7 @@ def quantile_with_mask(
interpolation=interpolation,
)
- result = np.array(result, copy=False)
+ result = np.asarray(result)
result = result.T
return result
@@ -201,9 +201,9 @@ def _nanpercentile(
]
if values.dtype.kind == "f":
# preserve itemsize
- result = np.array(result, dtype=values.dtype, copy=False).T
+ result = np.asarray(result, dtype=values.dtype).T
else:
- result = np.array(result, copy=False).T
+ result = np.asarray(result).T
if (
result.dtype != values.dtype
and not mask.all()
diff --git a/pandas/core/array_algos/replace.py b/pandas/core/array_algos/replace.py
index 60fc172139f13..f946c5adcbb0b 100644
--- a/pandas/core/array_algos/replace.py
+++ b/pandas/core/array_algos/replace.py
@@ -1,6 +1,7 @@
"""
Methods used by Block.replace and related methods.
"""
+
from __future__ import annotations
import operator
@@ -78,7 +79,7 @@ def _check_comparison_types(
type_names[0] = f"ndarray(dtype={a.dtype})"
raise TypeError(
- f"Cannot compare types {repr(type_names[0])} and {repr(type_names[1])}"
+ f"Cannot compare types {type_names[0]!r} and {type_names[1]!r}"
)
if not regex or not should_use_regex(regex, b):
@@ -92,17 +93,18 @@ def _check_comparison_types(
)
# GH#32621 use mask to avoid comparing to NAs
- if isinstance(a, np.ndarray):
+ if isinstance(a, np.ndarray) and mask is not None:
a = a[mask]
-
- result = op(a)
-
- if isinstance(result, np.ndarray) and mask is not None:
- # The shape of the mask can differ to that of the result
- # since we may compare only a subset of a's or b's elements
- tmp = np.zeros(mask.shape, dtype=np.bool_)
- np.place(tmp, mask, result)
- result = tmp
+ result = op(a)
+
+ if isinstance(result, np.ndarray):
+ # The shape of the mask can differ to that of the result
+ # since we may compare only a subset of a's or b's elements
+ tmp = np.zeros(mask.shape, dtype=np.bool_)
+ np.place(tmp, mask, result)
+ result = tmp
+ else:
+ result = op(a)
_check_comparison_types(result, a, b)
return result
diff --git a/pandas/core/array_algos/take.py b/pandas/core/array_algos/take.py
index ac674e31586e7..4d33b01f616cc 100644
--- a/pandas/core/array_algos/take.py
+++ b/pandas/core/array_algos/take.py
@@ -41,8 +41,7 @@ def take_nd(
axis: AxisInt = ...,
fill_value=...,
allow_fill: bool = ...,
-) -> np.ndarray:
- ...
+) -> np.ndarray: ...
@overload
@@ -52,8 +51,7 @@ def take_nd(
axis: AxisInt = ...,
fill_value=...,
allow_fill: bool = ...,
-) -> ArrayLike:
- ...
+) -> ArrayLike: ...
def take_nd(
@@ -166,64 +164,6 @@ def _take_nd_ndarray(
return out
-def take_1d(
- arr: ArrayLike,
- indexer: npt.NDArray[np.intp],
- fill_value=None,
- allow_fill: bool = True,
- mask: npt.NDArray[np.bool_] | None = None,
-) -> ArrayLike:
- """
- Specialized version for 1D arrays. Differences compared to `take_nd`:
-
- - Assumes input array has already been converted to numpy array / EA
- - Assumes indexer is already guaranteed to be intp dtype ndarray
- - Only works for 1D arrays
-
- To ensure the lowest possible overhead.
-
- Note: similarly to `take_nd`, this function assumes that the indexer is
- a valid(ated) indexer with no out of bound indices.
-
- Parameters
- ----------
- arr : np.ndarray or ExtensionArray
- Input array.
- indexer : ndarray
- 1-D array of indices to take (validated indices, intp dtype).
- fill_value : any, default np.nan
- Fill value to replace -1 values with
- allow_fill : bool, default True
- If False, indexer is assumed to contain no -1 values so no filling
- will be done. This short-circuits computation of a mask. Result is
- undefined if allow_fill == False and -1 is present in indexer.
- mask : np.ndarray, optional, default None
- If `allow_fill` is True, and the mask (where indexer == -1) is already
- known, it can be passed to avoid recomputation.
- """
- if not isinstance(arr, np.ndarray):
- # ExtensionArray -> dispatch to their method
- return arr.take(indexer, fill_value=fill_value, allow_fill=allow_fill)
-
- if not allow_fill:
- return arr.take(indexer)
-
- dtype, fill_value, mask_info = _take_preprocess_indexer_and_fill_value(
- arr, indexer, fill_value, True, mask
- )
-
- # at this point, it's guaranteed that dtype can hold both the arr values
- # and the fill_value
- out = np.empty(indexer.shape, dtype=dtype)
-
- func = _get_take_nd_function(
- arr.ndim, arr.dtype, out.dtype, axis=0, mask_info=mask_info
- )
- func(arr, indexer, out, fill_value)
-
- return out
-
-
def take_2d_multi(
arr: np.ndarray,
indexer: tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]],
@@ -397,6 +337,10 @@ def wrapper(
("int32", "int64"): libalgos.take_1d_int32_int64,
("int32", "float64"): libalgos.take_1d_int32_float64,
("int64", "int64"): libalgos.take_1d_int64_int64,
+ ("uint8", "uint8"): libalgos.take_1d_bool_bool,
+ ("uint16", "int64"): libalgos.take_1d_uint16_uint16,
+ ("uint32", "int64"): libalgos.take_1d_uint32_uint32,
+ ("uint64", "int64"): libalgos.take_1d_uint64_uint64,
("int64", "float64"): libalgos.take_1d_int64_float64,
("float32", "float32"): libalgos.take_1d_float32_float32,
("float32", "float64"): libalgos.take_1d_float32_float64,
@@ -426,6 +370,10 @@ def wrapper(
("int32", "float64"): libalgos.take_2d_axis0_int32_float64,
("int64", "int64"): libalgos.take_2d_axis0_int64_int64,
("int64", "float64"): libalgos.take_2d_axis0_int64_float64,
+ ("uint8", "uint8"): libalgos.take_2d_axis0_bool_bool,
+ ("uint16", "uint16"): libalgos.take_2d_axis0_uint16_uint16,
+ ("uint32", "uint32"): libalgos.take_2d_axis0_uint32_uint32,
+ ("uint64", "uint64"): libalgos.take_2d_axis0_uint64_uint64,
("float32", "float32"): libalgos.take_2d_axis0_float32_float32,
("float32", "float64"): libalgos.take_2d_axis0_float32_float64,
("float64", "float64"): libalgos.take_2d_axis0_float64_float64,
@@ -458,6 +406,10 @@ def wrapper(
("int32", "float64"): libalgos.take_2d_axis1_int32_float64,
("int64", "int64"): libalgos.take_2d_axis1_int64_int64,
("int64", "float64"): libalgos.take_2d_axis1_int64_float64,
+ ("uint8", "uint8"): libalgos.take_2d_axis1_bool_bool,
+ ("uint16", "uint16"): libalgos.take_2d_axis1_uint16_uint16,
+ ("uint32", "uint32"): libalgos.take_2d_axis1_uint32_uint32,
+ ("uint64", "uint64"): libalgos.take_2d_axis1_uint64_uint64,
("float32", "float32"): libalgos.take_2d_axis1_float32_float32,
("float32", "float64"): libalgos.take_2d_axis1_float32_float64,
("float64", "float64"): libalgos.take_2d_axis1_float64_float64,
diff --git a/pandas/core/arraylike.py b/pandas/core/arraylike.py
index fd585b3e19468..1fa610f35f56b 100644
--- a/pandas/core/arraylike.py
+++ b/pandas/core/arraylike.py
@@ -4,6 +4,7 @@
Index
ExtensionArray
"""
+
from __future__ import annotations
import operator
@@ -119,8 +120,9 @@ def __add__(self, other):
Examples
--------
- >>> df = pd.DataFrame({'height': [1.5, 2.6], 'weight': [500, 800]},
- ... index=['elk', 'moose'])
+ >>> df = pd.DataFrame(
+ ... {"height": [1.5, 2.6], "weight": [500, 800]}, index=["elk", "moose"]
+ ... )
>>> df
height weight
elk 1.5 500
@@ -128,14 +130,14 @@ def __add__(self, other):
Adding a scalar affects all rows and columns.
- >>> df[['height', 'weight']] + 1.5
+ >>> df[["height", "weight"]] + 1.5
height weight
elk 3.0 501.5
moose 4.1 801.5
Each element of a list is added to a column of the DataFrame, in order.
- >>> df[['height', 'weight']] + [0.5, 1.5]
+ >>> df[["height", "weight"]] + [0.5, 1.5]
height weight
elk 2.0 501.5
moose 3.1 801.5
@@ -143,7 +145,7 @@ def __add__(self, other):
Keys of a dictionary are aligned to the DataFrame, based on column names;
each value in the dictionary is added to the corresponding column.
- >>> df[['height', 'weight']] + {'height': 0.5, 'weight': 1.5}
+ >>> df[["height", "weight"]] + {"height": 0.5, "weight": 1.5}
height weight
elk 2.0 501.5
moose 3.1 801.5
@@ -151,8 +153,8 @@ def __add__(self, other):
When `other` is a :class:`Series`, the index of `other` is aligned with the
columns of the DataFrame.
- >>> s1 = pd.Series([0.5, 1.5], index=['weight', 'height'])
- >>> df[['height', 'weight']] + s1
+ >>> s1 = pd.Series([0.5, 1.5], index=["weight", "height"])
+ >>> df[["height", "weight"]] + s1
height weight
elk 3.0 500.5
moose 4.1 800.5
@@ -161,13 +163,13 @@ def __add__(self, other):
the :class:`Series` will not be reoriented. If index-wise alignment is desired,
:meth:`DataFrame.add` should be used with `axis='index'`.
- >>> s2 = pd.Series([0.5, 1.5], index=['elk', 'moose'])
- >>> df[['height', 'weight']] + s2
+ >>> s2 = pd.Series([0.5, 1.5], index=["elk", "moose"])
+ >>> df[["height", "weight"]] + s2
elk height moose weight
elk NaN NaN NaN NaN
moose NaN NaN NaN NaN
- >>> df[['height', 'weight']].add(s2, axis='index')
+ >>> df[["height", "weight"]].add(s2, axis="index")
height weight
elk 2.0 500.5
moose 4.1 801.5
@@ -175,9 +177,10 @@ def __add__(self, other):
When `other` is a :class:`DataFrame`, both columns names and the
index are aligned.
- >>> other = pd.DataFrame({'height': [0.2, 0.4, 0.6]},
- ... index=['elk', 'moose', 'deer'])
- >>> df[['height', 'weight']] + other
+ >>> other = pd.DataFrame(
+ ... {"height": [0.2, 0.4, 0.6]}, index=["elk", "moose", "deer"]
+ ... )
+ >>> df[["height", "weight"]] + other
height weight
deer NaN NaN
elk 1.7 NaN
@@ -263,10 +266,7 @@ def array_ufunc(self, ufunc: np.ufunc, method: str, *inputs: Any, **kwargs: Any)
Series,
)
from pandas.core.generic import NDFrame
- from pandas.core.internals import (
- ArrayManager,
- BlockManager,
- )
+ from pandas.core.internals import BlockManager
cls = type(self)
@@ -350,7 +350,7 @@ def _reconstruct(result):
if method == "outer":
raise NotImplementedError
return result
- if isinstance(result, (BlockManager, ArrayManager)):
+ if isinstance(result, BlockManager):
# we went through BlockManager.apply e.g. np.sqrt
result = self._constructor_from_mgr(result, axes=result.axes)
else:
diff --git a/pandas/core/arrays/_arrow_string_mixins.py b/pandas/core/arrays/_arrow_string_mixins.py
index bfff19a123a08..06c74290bd82e 100644
--- a/pandas/core/arrays/_arrow_string_mixins.py
+++ b/pandas/core/arrays/_arrow_string_mixins.py
@@ -58,7 +58,8 @@ def _str_get(self, i: int) -> Self:
self._pa_array, start=start, stop=stop, step=step
)
null_value = pa.scalar(
- None, type=self._pa_array.type # type: ignore[attr-defined]
+ None,
+ type=self._pa_array.type, # type: ignore[attr-defined]
)
result = pc.if_else(not_out_of_bounds, selected, null_value)
return type(self)(result)
diff --git a/pandas/core/arrays/_mixins.py b/pandas/core/arrays/_mixins.py
index 9ece12cf51a7b..7f4e6f6666382 100644
--- a/pandas/core/arrays/_mixins.py
+++ b/pandas/core/arrays/_mixins.py
@@ -33,7 +33,6 @@
from pandas.util._decorators import doc
from pandas.util._validators import (
validate_bool_kwarg,
- validate_fillna_kwargs,
validate_insert_loc,
)
@@ -89,7 +88,9 @@ def method(self, *args, **kwargs):
return cast(F, method)
-class NDArrayBackedExtensionArray(NDArrayBacked, ExtensionArray):
+# error: Definition of "delete/ravel/T/repeat/copy" in base class "NDArrayBacked"
+# is incompatible with definition in base class "ExtensionArray"
+class NDArrayBackedExtensionArray(NDArrayBacked, ExtensionArray): # type: ignore[misc]
"""
ExtensionArray that is backed by a single NumPy ndarray.
"""
@@ -248,7 +249,7 @@ def searchsorted(
return self._ndarray.searchsorted(npvalue, side=side, sorter=sorter)
@doc(ExtensionArray.shift)
- def shift(self, periods: int = 1, fill_value=None):
+ def shift(self, periods: int = 1, fill_value=None) -> Self:
# NB: shift is always along axis=0
axis = 0
fill_value = self._validate_scalar(fill_value)
@@ -265,15 +266,13 @@ def _validate_setitem_value(self, value):
return value
@overload
- def __getitem__(self, key: ScalarIndexer) -> Any:
- ...
+ def __getitem__(self, key: ScalarIndexer) -> Any: ...
@overload
def __getitem__(
self,
key: SequenceIndexer | PositionalIndexerTuple,
- ) -> Self:
- ...
+ ) -> Self: ...
def __getitem__(
self,
@@ -305,7 +304,12 @@ def _fill_mask_inplace(
func(self._ndarray.T, limit=limit, mask=mask.T)
def _pad_or_backfill(
- self, *, method: FillnaOptions, limit: int | None = None, copy: bool = True
+ self,
+ *,
+ method: FillnaOptions,
+ limit: int | None = None,
+ limit_area: Literal["inside", "outside"] | None = None,
+ copy: bool = True,
) -> Self:
mask = self.isna()
if mask.any():
@@ -315,7 +319,7 @@ def _pad_or_backfill(
npvalues = self._ndarray.T
if copy:
npvalues = npvalues.copy()
- func(npvalues, limit=limit, mask=mask.T)
+ func(npvalues, limit=limit, limit_area=limit_area, mask=mask.T)
npvalues = npvalues.T
if copy:
@@ -331,40 +335,23 @@ def _pad_or_backfill(
return new_values
@doc(ExtensionArray.fillna)
- def fillna(
- self, value=None, method=None, limit: int | None = None, copy: bool = True
- ) -> Self:
- value, method = validate_fillna_kwargs(
- value, method, validate_scalar_dict_value=False
- )
-
+ def fillna(self, value=None, limit: int | None = None, copy: bool = True) -> Self:
mask = self.isna()
# error: Argument 2 to "check_value_size" has incompatible type
# "ExtensionArray"; expected "ndarray"
value = missing.check_value_size(
- value, mask, len(self) # type: ignore[arg-type]
+ value,
+ mask, # type: ignore[arg-type]
+ len(self),
)
if mask.any():
- if method is not None:
- # (for now) when self.ndim == 2, we assume axis=0
- func = missing.get_fill_func(method, ndim=self.ndim)
- npvalues = self._ndarray.T
- if copy:
- npvalues = npvalues.copy()
- func(npvalues, limit=limit, mask=mask.T)
- npvalues = npvalues.T
-
- # TODO: NumpyExtensionArray didn't used to copy, need tests
- # for this
- new_values = self._from_backing_data(npvalues)
+ # fill with value
+ if copy:
+ new_values = self.copy()
else:
- # fill with value
- if copy:
- new_values = self.copy()
- else:
- new_values = self[:]
- new_values[mask] = value
+ new_values = self[:]
+ new_values[mask] = value
else:
# We validate the fill_value even if there is nothing to fill
if value is not None:
@@ -379,7 +366,7 @@ def fillna(
# ------------------------------------------------------------------------
# Reductions
- def _wrap_reduction_result(self, axis: AxisInt | None, result):
+ def _wrap_reduction_result(self, axis: AxisInt | None, result) -> Any:
if axis is None or self.ndim == 1:
return self._box_func(result)
return self._from_backing_data(result)
diff --git a/pandas/core/arrays/_ranges.py b/pandas/core/arrays/_ranges.py
index 3e89391324ad4..600ddc7f717a8 100644
--- a/pandas/core/arrays/_ranges.py
+++ b/pandas/core/arrays/_ranges.py
@@ -2,6 +2,7 @@
Helper functions to generate range-like data for DatetimeArray
(and possibly TimedeltaArray/PeriodArray)
"""
+
from __future__ import annotations
from typing import TYPE_CHECKING
diff --git a/pandas/core/arrays/_utils.py b/pandas/core/arrays/_utils.py
index c75ec7f843ed2..6b46396d5efdf 100644
--- a/pandas/core/arrays/_utils.py
+++ b/pandas/core/arrays/_utils.py
@@ -44,7 +44,16 @@ def to_numpy_dtype_inference(
dtype_given = True
if na_value is lib.no_default:
- na_value = arr.dtype.na_value
+ if dtype is None or not hasna:
+ na_value = arr.dtype.na_value
+ elif dtype.kind == "f": # type: ignore[union-attr]
+ na_value = np.nan
+ elif dtype.kind == "M": # type: ignore[union-attr]
+ na_value = np.datetime64("nat")
+ elif dtype.kind == "m": # type: ignore[union-attr]
+ na_value = np.timedelta64("nat")
+ else:
+ na_value = arr.dtype.na_value
if not dtype_given and hasna:
try:
diff --git a/pandas/core/arrays/arrow/_arrow_utils.py b/pandas/core/arrays/arrow/_arrow_utils.py
index 2a053fac2985c..cbc9ce0252750 100644
--- a/pandas/core/arrays/arrow/_arrow_utils.py
+++ b/pandas/core/arrays/arrow/_arrow_utils.py
@@ -5,6 +5,8 @@
import numpy as np
import pyarrow
+from pandas._config.config import get_option
+
from pandas.errors import PerformanceWarning
from pandas.util._exceptions import find_stack_level
@@ -14,10 +16,11 @@ def fallback_performancewarning(version: str | None = None) -> None:
Raise a PerformanceWarning for falling back to ExtensionArray's
non-pyarrow method
"""
- msg = "Falling back on a non-pyarrow code path which may decrease performance."
- if version is not None:
- msg += f" Upgrade to pyarrow >={version} to possibly suppress this warning."
- warnings.warn(msg, PerformanceWarning, stacklevel=find_stack_level())
+ if get_option("performance_warnings"):
+ msg = "Falling back on a non-pyarrow code path which may decrease performance."
+ if version is not None:
+ msg += f" Upgrade to pyarrow >={version} to possibly suppress this warning."
+ warnings.warn(msg, PerformanceWarning, stacklevel=find_stack_level())
def pyarrow_array_to_numpy_and_mask(
diff --git a/pandas/core/arrays/arrow/accessors.py b/pandas/core/arrays/arrow/accessors.py
index 23825faa70095..19ec253e81ef2 100644
--- a/pandas/core/arrays/arrow/accessors.py
+++ b/pandas/core/arrays/arrow/accessors.py
@@ -6,13 +6,18 @@
ABCMeta,
abstractmethod,
)
-from typing import TYPE_CHECKING
+from typing import (
+ TYPE_CHECKING,
+ cast,
+)
from pandas.compat import (
pa_version_under10p1,
pa_version_under11p0,
)
+from pandas.core.dtypes.common import is_list_like
+
if not pa_version_under10p1:
import pyarrow as pa
import pyarrow.compute as pc
@@ -95,9 +100,7 @@ def len(self) -> Series:
... [1, 2, 3],
... [3],
... ],
- ... dtype=pd.ArrowDtype(pa.list_(
- ... pa.int64()
- ... ))
+ ... dtype=pd.ArrowDtype(pa.list_(pa.int64())),
... )
>>> s.list.len()
0 3
@@ -131,9 +134,7 @@ def __getitem__(self, key: int | slice) -> Series:
... [1, 2, 3],
... [3],
... ],
- ... dtype=pd.ArrowDtype(pa.list_(
- ... pa.int64()
- ... ))
+ ... dtype=pd.ArrowDtype(pa.list_(pa.int64())),
... )
>>> s.list[0]
0 1
@@ -190,9 +191,7 @@ def flatten(self) -> Series:
... [1, 2, 3],
... [3],
... ],
- ... dtype=pd.ArrowDtype(pa.list_(
- ... pa.int64()
- ... ))
+ ... dtype=pd.ArrowDtype(pa.list_(pa.int64())),
... )
>>> s.list.flatten()
0 1
@@ -248,9 +247,9 @@ def dtypes(self) -> Series:
... {"version": 2, "project": "pandas"},
... {"version": 1, "project": "numpy"},
... ],
- ... dtype=pd.ArrowDtype(pa.struct(
- ... [("version", pa.int64()), ("project", pa.string())]
- ... ))
+ ... dtype=pd.ArrowDtype(
+ ... pa.struct([("version", pa.int64()), ("project", pa.string())])
+ ... ),
... )
>>> s.struct.dtypes
version int64[pyarrow]
@@ -267,15 +266,27 @@ def dtypes(self) -> Series:
names = [struct.name for struct in pa_type]
return Series(types, index=Index(names))
- def field(self, name_or_index: str | int) -> Series:
+ def field(
+ self,
+ name_or_index: list[str]
+ | list[bytes]
+ | list[int]
+ | pc.Expression
+ | bytes
+ | str
+ | int,
+ ) -> Series:
"""
Extract a child field of a struct as a Series.
Parameters
----------
- name_or_index : str | int
+ name_or_index : str | bytes | int | expression | list
Name or index of the child field to extract.
+ For list-like inputs, this will index into a nested
+ struct.
+
Returns
-------
pandas.Series
@@ -285,6 +296,19 @@ def field(self, name_or_index: str | int) -> Series:
--------
Series.struct.explode : Return all child fields as a DataFrame.
+ Notes
+ -----
+ The name of the resulting Series will be set using the following
+ rules:
+
+ - For string, bytes, or integer `name_or_index` (or a list of these, for
+ a nested selection), the Series name is set to the selected
+ field's name.
+ - For a :class:`pyarrow.compute.Expression`, this is set to
+ the string form of the expression.
+ - For list-like `name_or_index`, the name will be set to the
+ name of the final field selected.
+
Examples
--------
>>> import pyarrow as pa
@@ -294,9 +318,9 @@ def field(self, name_or_index: str | int) -> Series:
... {"version": 2, "project": "pandas"},
... {"version": 1, "project": "numpy"},
... ],
- ... dtype=pd.ArrowDtype(pa.struct(
- ... [("version", pa.int64()), ("project", pa.string())]
- ... ))
+ ... dtype=pd.ArrowDtype(
+ ... pa.struct([("version", pa.int64()), ("project", pa.string())])
+ ... ),
... )
Extract by field name.
@@ -314,27 +338,94 @@ def field(self, name_or_index: str | int) -> Series:
1 2
2 1
Name: version, dtype: int64[pyarrow]
+
+ Or an expression
+
+ >>> import pyarrow.compute as pc
+ >>> s.struct.field(pc.field("project"))
+ 0 pandas
+ 1 pandas
+ 2 numpy
+ Name: project, dtype: string[pyarrow]
+
+ For nested struct types, you can pass a list of values to index
+ multiple levels:
+
+ >>> version_type = pa.struct(
+ ... [
+ ... ("major", pa.int64()),
+ ... ("minor", pa.int64()),
+ ... ]
+ ... )
+ >>> s = pd.Series(
+ ... [
+ ... {"version": {"major": 1, "minor": 5}, "project": "pandas"},
+ ... {"version": {"major": 2, "minor": 1}, "project": "pandas"},
+ ... {"version": {"major": 1, "minor": 26}, "project": "numpy"},
+ ... ],
+ ... dtype=pd.ArrowDtype(
+ ... pa.struct([("version", version_type), ("project", pa.string())])
+ ... ),
+ ... )
+ >>> s.struct.field(["version", "minor"])
+ 0 5
+ 1 1
+ 2 26
+ Name: minor, dtype: int64[pyarrow]
+ >>> s.struct.field([0, 0])
+ 0 1
+ 1 2
+ 2 1
+ Name: major, dtype: int64[pyarrow]
"""
from pandas import Series
+ def get_name(
+ level_name_or_index: list[str]
+ | list[bytes]
+ | list[int]
+ | pc.Expression
+ | bytes
+ | str
+ | int,
+ data: pa.ChunkedArray,
+ ):
+ if isinstance(level_name_or_index, int):
+ name = data.type.field(level_name_or_index).name
+ elif isinstance(level_name_or_index, (str, bytes)):
+ name = level_name_or_index
+ elif isinstance(level_name_or_index, pc.Expression):
+ name = str(level_name_or_index)
+ elif is_list_like(level_name_or_index):
+ # For nested input like [2, 1, 2]
+ # iteratively get the struct and field name. The last
+ # one is used for the name of the index.
+ level_name_or_index = list(reversed(level_name_or_index))
+ selected = data
+ while level_name_or_index:
+ # we need the cast, otherwise mypy complains about
+ # getting ints, bytes, or str here, which isn't possible.
+ level_name_or_index = cast(list, level_name_or_index)
+ name_or_index = level_name_or_index.pop()
+ name = get_name(name_or_index, selected)
+ selected = selected.type.field(selected.type.get_field_index(name))
+ name = selected.name
+ else:
+ raise ValueError(
+ "name_or_index must be an int, str, bytes, "
+ "pyarrow.compute.Expression, or list of those"
+ )
+ return name
+
pa_arr = self._data.array._pa_array
- if isinstance(name_or_index, int):
- index = name_or_index
- elif isinstance(name_or_index, str):
- index = pa_arr.type.get_field_index(name_or_index)
- else:
- raise ValueError(
- "name_or_index must be an int or str, "
- f"got {type(name_or_index).__name__}"
- )
+ name = get_name(name_or_index, pa_arr)
+ field_arr = pc.struct_field(pa_arr, name_or_index)
- pa_field = pa_arr.type[index]
- field_arr = pc.struct_field(pa_arr, [index])
return Series(
field_arr,
dtype=ArrowDtype(field_arr.type),
index=self._data.index,
- name=pa_field.name,
+ name=name,
)
def explode(self) -> DataFrame:
@@ -359,9 +450,9 @@ def explode(self) -> DataFrame:
... {"version": 2, "project": "pandas"},
... {"version": 1, "project": "numpy"},
... ],
- ... dtype=pd.ArrowDtype(pa.struct(
- ... [("version", pa.int64()), ("project", pa.string())]
- ... ))
+ ... dtype=pd.ArrowDtype(
+ ... pa.struct([("version", pa.int64()), ("project", pa.string())])
+ ... ),
... )
>>> s.struct.explode()
diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py
index b1164301e6d79..84b62563605ac 100644
--- a/pandas/core/arrays/arrow/array.py
+++ b/pandas/core/arrays/arrow/array.py
@@ -10,6 +10,7 @@
Callable,
Literal,
cast,
+ overload,
)
import unicodedata
@@ -28,7 +29,6 @@
pa_version_under13p0,
)
from pandas.util._decorators import doc
-from pandas.util._validators import validate_fillna_kwargs
from pandas.core.dtypes.cast import (
can_hold_element,
@@ -109,30 +109,50 @@
def cast_for_truediv(
arrow_array: pa.ChunkedArray, pa_object: pa.Array | pa.Scalar
- ) -> pa.ChunkedArray:
+ ) -> tuple[pa.ChunkedArray, pa.Array | pa.Scalar]:
# Ensure int / int -> float mirroring Python/Numpy behavior
# as pc.divide_checked(int, int) -> int
if pa.types.is_integer(arrow_array.type) and pa.types.is_integer(
pa_object.type
):
+ # GH: 56645.
# https://github.com/apache/arrow/issues/35563
- # Arrow does not allow safe casting large integral values to float64.
- # Intentionally not using arrow_array.cast because it could be a scalar
- # value in reflected case, and safe=False only added to
- # scalar cast in pyarrow 13.
- return pc.cast(arrow_array, pa.float64(), safe=False)
- return arrow_array
+ return pc.cast(arrow_array, pa.float64(), safe=False), pc.cast(
+ pa_object, pa.float64(), safe=False
+ )
+
+ return arrow_array, pa_object
def floordiv_compat(
left: pa.ChunkedArray | pa.Array | pa.Scalar,
right: pa.ChunkedArray | pa.Array | pa.Scalar,
) -> pa.ChunkedArray:
- # Ensure int // int -> int mirroring Python/Numpy behavior
- # as pc.floor(pc.divide_checked(int, int)) -> float
- converted_left = cast_for_truediv(left, right)
- result = pc.floor(pc.divide(converted_left, right))
+ # TODO: Replace with pyarrow floordiv kernel.
+ # https://github.com/apache/arrow/issues/39386
if pa.types.is_integer(left.type) and pa.types.is_integer(right.type):
+ divided = pc.divide_checked(left, right)
+ if pa.types.is_signed_integer(divided.type):
+ # GH 56676
+ has_remainder = pc.not_equal(pc.multiply(divided, right), left)
+ has_one_negative_operand = pc.less(
+ pc.bit_wise_xor(left, right),
+ pa.scalar(0, type=divided.type),
+ )
+ result = pc.if_else(
+ pc.and_(
+ has_remainder,
+ has_one_negative_operand,
+ ),
+ # GH: 55561
+ pc.subtract(divided, pa.scalar(1, type=divided.type)),
+ divided,
+ )
+ else:
+ result = divided
result = result.cast(left.type)
+ else:
+ divided = pc.divide(left, right)
+ result = pc.floor(divided)
return result
ARROW_ARITHMETIC_FUNCS = {
@@ -142,8 +162,8 @@ def floordiv_compat(
"rsub": lambda x, y: pc.subtract_checked(y, x),
"mul": pc.multiply_checked,
"rmul": lambda x, y: pc.multiply_checked(y, x),
- "truediv": lambda x, y: pc.divide(cast_for_truediv(x, y), y),
- "rtruediv": lambda x, y: pc.divide(y, cast_for_truediv(x, y)),
+ "truediv": lambda x, y: pc.divide(*cast_for_truediv(x, y)),
+ "rtruediv": lambda x, y: pc.divide(*cast_for_truediv(y, x)),
"floordiv": lambda x, y: floordiv_compat(x, y),
"rfloordiv": lambda x, y: floordiv_compat(y, x),
"mod": NotImplemented,
@@ -157,11 +177,13 @@ def floordiv_compat(
if TYPE_CHECKING:
from collections.abc import Sequence
+ from pandas._libs.missing import NAType
from pandas._typing import (
ArrayLike,
AxisInt,
Dtype,
FillnaOptions,
+ InterpolateOptions,
Iterator,
NpDtype,
NumpySorter,
@@ -176,12 +198,14 @@ def floordiv_compat(
npt,
)
+ from pandas.core.dtypes.dtypes import ExtensionDtype
+
from pandas import Series
from pandas.core.arrays.datetimes import DatetimeArray
from pandas.core.arrays.timedeltas import TimedeltaArray
-def get_unit_from_pa_dtype(pa_dtype):
+def get_unit_from_pa_dtype(pa_dtype) -> str:
# https://github.com/pandas-dev/pandas/pull/50998#discussion_r1100344804
if pa_version_under11p0:
unit = str(pa_dtype).split("[", 1)[-1][:-1]
@@ -280,7 +304,9 @@ def __init__(self, values: pa.Array | pa.ChunkedArray) -> None:
self._dtype = ArrowDtype(self._pa_array.type)
@classmethod
- def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
+ def _from_sequence(
+ cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
+ ) -> Self:
"""
Construct a new ExtensionArray from a sequence of scalars.
"""
@@ -291,8 +317,8 @@ def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = Fal
@classmethod
def _from_sequence_of_strings(
- cls, strings, *, dtype: Dtype | None = None, copy: bool = False
- ):
+ cls, strings, *, dtype: ExtensionDtype, copy: bool = False
+ ) -> Self:
"""
Construct a new ExtensionArray from a sequence of strings.
"""
@@ -502,18 +528,15 @@ def _box_pa_array(
else:
try:
pa_array = pa_array.cast(pa_type)
- except (
- pa.ArrowInvalid,
- pa.ArrowTypeError,
- pa.ArrowNotImplementedError,
- ):
+ except (pa.ArrowNotImplementedError, pa.ArrowTypeError):
if pa.types.is_string(pa_array.type) or pa.types.is_large_string(
pa_array.type
):
# TODO: Move logic in _from_sequence_of_strings into
# _box_pa_array
+ dtype = ArrowDtype(pa_type)
return cls._from_sequence_of_strings(
- value, dtype=pa_type
+ value, dtype=dtype
)._pa_array
else:
raise
@@ -635,7 +658,9 @@ def __arrow_array__(self, type=None):
"""Convert myself to a pyarrow ChunkedArray."""
return self._pa_array
- def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+ def __array__(
+ self, dtype: NpDtype | None = None, copy: bool | None = None
+ ) -> np.ndarray:
"""Correctly construct numpy arrays when passed to `np.asarray()`."""
return self.to_numpy(dtype=dtype)
@@ -675,7 +700,7 @@ def __setstate__(self, state) -> None:
state["_pa_array"] = pa.chunked_array(data)
self.__dict__.update(state)
- def _cmp_method(self, other, op):
+ def _cmp_method(self, other, op) -> ArrowExtensionArray:
pc_func = ARROW_CMP_FUNCS[op.__name__]
if isinstance(
other, (ArrowExtensionArray, np.ndarray, list, BaseMaskedArray)
@@ -701,7 +726,7 @@ def _cmp_method(self, other, op):
)
return ArrowExtensionArray(result)
- def _evaluate_op_method(self, other, op, arrow_funcs):
+ def _evaluate_op_method(self, other, op, arrow_funcs) -> Self:
pa_type = self._pa_array.type
other = self._box_pa(other)
@@ -752,7 +777,7 @@ def _evaluate_op_method(self, other, op, arrow_funcs):
result = pc_func(self._pa_array, other)
return type(self)(result)
- def _logical_method(self, other, op):
+ def _logical_method(self, other, op) -> Self:
# For integer types `^`, `|`, `&` are bitwise operators and return
# integer types. Otherwise these are boolean ops.
if pa.types.is_integer(self._pa_array.type):
@@ -760,7 +785,7 @@ def _logical_method(self, other, op):
else:
return self._evaluate_op_method(other, op, ARROW_LOGICAL_FUNCS)
- def _arith_method(self, other, op):
+ def _arith_method(self, other, op) -> Self:
return self._evaluate_op_method(other, op, ARROW_ARITHMETIC_FUNCS)
def equals(self, other) -> bool:
@@ -825,7 +850,13 @@ def isna(self) -> npt.NDArray[np.bool_]:
return self._pa_array.is_null().to_numpy()
- def any(self, *, skipna: bool = True, **kwargs):
+ @overload
+ def any(self, *, skipna: Literal[True] = ..., **kwargs) -> bool: ...
+
+ @overload
+ def any(self, *, skipna: bool, **kwargs) -> bool | NAType: ...
+
+ def any(self, *, skipna: bool = True, **kwargs) -> bool | NAType:
"""
Return whether any element is truthy.
@@ -883,7 +914,13 @@ def any(self, *, skipna: bool = True, **kwargs):
"""
return self._reduce("any", skipna=skipna, **kwargs)
- def all(self, *, skipna: bool = True, **kwargs):
+ @overload
+ def all(self, *, skipna: Literal[True] = ..., **kwargs) -> bool: ...
+
+ @overload
+ def all(self, *, skipna: bool, **kwargs) -> bool | NAType: ...
+
+ def all(self, *, skipna: bool = True, **kwargs) -> bool | NAType:
"""
Return whether all elements are truthy.
@@ -1005,13 +1042,18 @@ def dropna(self) -> Self:
return type(self)(pc.drop_null(self._pa_array))
def _pad_or_backfill(
- self, *, method: FillnaOptions, limit: int | None = None, copy: bool = True
+ self,
+ *,
+ method: FillnaOptions,
+ limit: int | None = None,
+ limit_area: Literal["inside", "outside"] | None = None,
+ copy: bool = True,
) -> Self:
if not self._hasna:
# TODO(CoW): Not necessary anymore when CoW is the default
return self.copy()
- if limit is None:
+ if limit is None and limit_area is None:
method = missing.clean_fill_method(method)
try:
if method == "pad":
@@ -1025,29 +1067,26 @@ def _pad_or_backfill(
# a kernel for duration types.
pass
+ # TODO: Why do we no longer need the above cases?
# TODO(3.0): after EA.fillna 'method' deprecation is enforced, we can remove
# this method entirely.
- return super()._pad_or_backfill(method=method, limit=limit, copy=copy)
+ return super()._pad_or_backfill(
+ method=method, limit=limit, limit_area=limit_area, copy=copy
+ )
@doc(ExtensionArray.fillna)
def fillna(
self,
value: object | ArrayLike | None = None,
- method: FillnaOptions | None = None,
limit: int | None = None,
copy: bool = True,
) -> Self:
- value, method = validate_fillna_kwargs(value, method)
-
if not self._hasna:
# TODO(CoW): Not necessary anymore when CoW is the default
return self.copy()
if limit is not None:
- return super().fillna(value=value, method=method, limit=limit, copy=copy)
-
- if method is not None:
- return super().fillna(method=method, limit=limit, copy=copy)
+ return super().fillna(value=value, limit=limit, copy=copy)
if isinstance(value, (np.ndarray, ExtensionArray)):
# Similar to check_value_size, but we do not mask here since we may
@@ -1061,7 +1100,7 @@ def fillna(
try:
fill_value = self._box_pa(value, pa_type=self._pa_array.type)
except pa.ArrowTypeError as err:
- msg = f"Invalid value '{str(value)}' for dtype {self.dtype}"
+ msg = f"Invalid value '{value!s}' for dtype {self.dtype}"
raise TypeError(msg) from err
try:
@@ -1073,7 +1112,7 @@ def fillna(
# a kernel for duration types.
pass
- return super().fillna(value=value, method=method, limit=limit, copy=copy)
+ return super().fillna(value=value, limit=limit, copy=copy)
def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]:
# short-circuit to return all False array.
@@ -1320,6 +1359,11 @@ def _to_timedeltaarray(self) -> TimedeltaArray:
np_array = np_array.astype(np_dtype)
return TimedeltaArray._simple_new(np_array, dtype=np_dtype)
+ def _values_for_json(self) -> np.ndarray:
+ if is_numeric_dtype(self.dtype):
+ return np.asarray(self, dtype=object)
+ return super()._values_for_json()
+
@doc(ExtensionArray.to_numpy)
def to_numpy(
self,
@@ -1381,7 +1425,7 @@ def to_numpy(
def map(self, mapper, na_action=None):
if is_numeric_dtype(self.dtype):
- return map_array(self.to_numpy(), mapper, na_action=None)
+ return map_array(self.to_numpy(), mapper, na_action=na_action)
else:
return super().map(mapper, na_action)
@@ -1917,7 +1961,7 @@ def _rank(
na_option: str = "keep",
ascending: bool = True,
pct: bool = False,
- ):
+ ) -> Self:
"""
See Series.rank.__doc__.
"""
@@ -2017,17 +2061,73 @@ def _maybe_convert_setitem_value(self, value):
try:
value = self._box_pa(value, self._pa_array.type)
except pa.ArrowTypeError as err:
- msg = f"Invalid value '{str(value)}' for dtype {self.dtype}"
+ msg = f"Invalid value '{value!s}' for dtype {self.dtype}"
raise TypeError(msg) from err
return value
+ def interpolate(
+ self,
+ *,
+ method: InterpolateOptions,
+ axis: int,
+ index,
+ limit,
+ limit_direction,
+ limit_area,
+ copy: bool,
+ **kwargs,
+ ) -> Self:
+ """
+ See NDFrame.interpolate.__doc__.
+ """
+ # NB: we return type(self) even if copy=False
+ if not self.dtype._is_numeric:
+ raise ValueError("Values must be numeric.")
+
+ if (
+ not pa_version_under13p0
+ and method == "linear"
+ and limit_area is None
+ and limit is None
+ and limit_direction == "forward"
+ ):
+ values = self._pa_array.combine_chunks()
+ na_value = pa.array([None], type=values.type)
+ y_diff_2 = pc.fill_null_backward(pc.pairwise_diff_checked(values, period=2))
+ prev_values = pa.concat_arrays([na_value, values[:-2], na_value])
+ interps = pc.add_checked(prev_values, pc.divide_checked(y_diff_2, 2))
+ return type(self)(pc.coalesce(self._pa_array, interps))
+
+ mask = self.isna()
+ if self.dtype.kind == "f":
+ data = self._pa_array.to_numpy()
+ elif self.dtype.kind in "iu":
+ data = self.to_numpy(dtype="f8", na_value=0.0)
+ else:
+ raise NotImplementedError(
+ f"interpolate is not implemented for dtype={self.dtype}"
+ )
+
+ missing.interpolate_2d_inplace(
+ data,
+ method=method,
+ axis=0,
+ index=index,
+ limit=limit,
+ limit_direction=limit_direction,
+ limit_area=limit_area,
+ mask=mask,
+ **kwargs,
+ )
+ return type(self)(self._box_pa_array(pa.array(data, mask=mask)))
+
@classmethod
def _if_else(
cls,
cond: npt.NDArray[np.bool_] | bool,
left: ArrayLike | Scalar,
right: ArrayLike | Scalar,
- ):
+ ) -> pa.Array:
"""
Choose values based on a condition.
@@ -2071,7 +2171,7 @@ def _replace_with_mask(
values: pa.Array | pa.ChunkedArray,
mask: npt.NDArray[np.bool_] | bool,
replacements: ArrayLike | Scalar,
- ):
+ ) -> pa.Array | pa.ChunkedArray:
"""
Replace items selected with a mask.
@@ -2178,14 +2278,14 @@ def _apply_elementwise(self, func: Callable) -> list[list[Any]]:
for chunk in self._pa_array.iterchunks()
]
- def _str_count(self, pat: str, flags: int = 0):
+ def _str_count(self, pat: str, flags: int = 0) -> Self:
if flags:
raise NotImplementedError(f"count not implemented with {flags=}")
return type(self)(pc.count_substring_regex(self._pa_array, pat))
def _str_contains(
self, pat, case: bool = True, flags: int = 0, na=None, regex: bool = True
- ):
+ ) -> Self:
if flags:
raise NotImplementedError(f"contains not implemented with {flags=}")
@@ -2198,7 +2298,7 @@ def _str_contains(
result = result.fill_null(na)
return type(self)(result)
- def _str_startswith(self, pat: str | tuple[str, ...], na=None):
+ def _str_startswith(self, pat: str | tuple[str, ...], na=None) -> Self:
if isinstance(pat, str):
result = pc.starts_with(self._pa_array, pattern=pat)
else:
@@ -2215,7 +2315,7 @@ def _str_startswith(self, pat: str | tuple[str, ...], na=None):
result = result.fill_null(na)
return type(self)(result)
- def _str_endswith(self, pat: str | tuple[str, ...], na=None):
+ def _str_endswith(self, pat: str | tuple[str, ...], na=None) -> Self:
if isinstance(pat, str):
result = pc.ends_with(self._pa_array, pattern=pat)
else:
@@ -2240,7 +2340,7 @@ def _str_replace(
case: bool = True,
flags: int = 0,
regex: bool = True,
- ):
+ ) -> Self:
if isinstance(pat, re.Pattern) or callable(repl) or not case or flags:
raise NotImplementedError(
"replace is not supported with a re.Pattern, callable repl, "
@@ -2259,46 +2359,51 @@ def _str_replace(
)
return type(self)(result)
- def _str_repeat(self, repeats: int | Sequence[int]):
+ def _str_repeat(self, repeats: int | Sequence[int]) -> Self:
if not isinstance(repeats, int):
raise NotImplementedError(
f"repeat is not implemented when repeats is {type(repeats).__name__}"
)
- else:
- return type(self)(pc.binary_repeat(self._pa_array, repeats))
+ return type(self)(pc.binary_repeat(self._pa_array, repeats))
def _str_match(
self, pat: str, case: bool = True, flags: int = 0, na: Scalar | None = None
- ):
+ ) -> Self:
if not pat.startswith("^"):
pat = f"^{pat}"
return self._str_contains(pat, case, flags, na, regex=True)
def _str_fullmatch(
self, pat, case: bool = True, flags: int = 0, na: Scalar | None = None
- ):
- if not pat.endswith("$") or pat.endswith("//$"):
+ ) -> Self:
+ if not pat.endswith("$") or pat.endswith("\\$"):
pat = f"{pat}$"
return self._str_match(pat, case, flags, na)
- def _str_find(self, sub: str, start: int = 0, end: int | None = None):
- if start != 0 and end is not None:
+ def _str_find(self, sub: str, start: int = 0, end: int | None = None) -> Self:
+ if (start == 0 or start is None) and end is None:
+ result = pc.find_substring(self._pa_array, sub)
+ else:
+ if sub == "":
+ # GH 56792
+ result = self._apply_elementwise(lambda val: val.find(sub, start, end))
+ return type(self)(pa.chunked_array(result))
+ if start is None:
+ start_offset = 0
+ start = 0
+ elif start < 0:
+ start_offset = pc.add(start, pc.utf8_length(self._pa_array))
+ start_offset = pc.if_else(pc.less(start_offset, 0), 0, start_offset)
+ else:
+ start_offset = start
slices = pc.utf8_slice_codeunits(self._pa_array, start, stop=end)
result = pc.find_substring(slices, sub)
- not_found = pc.equal(result, -1)
- start_offset = max(0, start)
+ found = pc.not_equal(result, pa.scalar(-1, type=result.type))
offset_result = pc.add(result, start_offset)
- result = pc.if_else(not_found, result, offset_result)
- elif start == 0 and end is None:
- slices = self._pa_array
- result = pc.find_substring(slices, sub)
- else:
- raise NotImplementedError(
- f"find not implemented with {sub=}, {start=}, {end=}"
- )
+ result = pc.if_else(found, offset_result, -1)
return type(self)(result)
- def _str_join(self, sep: str):
+ def _str_join(self, sep: str) -> Self:
if pa.types.is_string(self._pa_array.type) or pa.types.is_large_string(
self._pa_array.type
):
@@ -2308,19 +2413,19 @@ def _str_join(self, sep: str):
result = self._pa_array
return type(self)(pc.binary_join(result, sep))
- def _str_partition(self, sep: str, expand: bool):
+ def _str_partition(self, sep: str, expand: bool) -> Self:
predicate = lambda val: val.partition(sep)
result = self._apply_elementwise(predicate)
return type(self)(pa.chunked_array(result))
- def _str_rpartition(self, sep: str, expand: bool):
+ def _str_rpartition(self, sep: str, expand: bool) -> Self:
predicate = lambda val: val.rpartition(sep)
result = self._apply_elementwise(predicate)
return type(self)(pa.chunked_array(result))
def _str_slice(
self, start: int | None = None, stop: int | None = None, step: int | None = None
- ):
+ ) -> Self:
if start is None:
start = 0
if step is None:
@@ -2329,57 +2434,57 @@ def _str_slice(
pc.utf8_slice_codeunits(self._pa_array, start=start, stop=stop, step=step)
)
- def _str_isalnum(self):
+ def _str_isalnum(self) -> Self:
return type(self)(pc.utf8_is_alnum(self._pa_array))
- def _str_isalpha(self):
+ def _str_isalpha(self) -> Self:
return type(self)(pc.utf8_is_alpha(self._pa_array))
- def _str_isdecimal(self):
+ def _str_isdecimal(self) -> Self:
return type(self)(pc.utf8_is_decimal(self._pa_array))
- def _str_isdigit(self):
+ def _str_isdigit(self) -> Self:
return type(self)(pc.utf8_is_digit(self._pa_array))
- def _str_islower(self):
+ def _str_islower(self) -> Self:
return type(self)(pc.utf8_is_lower(self._pa_array))
- def _str_isnumeric(self):
+ def _str_isnumeric(self) -> Self:
return type(self)(pc.utf8_is_numeric(self._pa_array))
- def _str_isspace(self):
+ def _str_isspace(self) -> Self:
return type(self)(pc.utf8_is_space(self._pa_array))
- def _str_istitle(self):
+ def _str_istitle(self) -> Self:
return type(self)(pc.utf8_is_title(self._pa_array))
- def _str_isupper(self):
+ def _str_isupper(self) -> Self:
return type(self)(pc.utf8_is_upper(self._pa_array))
- def _str_len(self):
+ def _str_len(self) -> Self:
return type(self)(pc.utf8_length(self._pa_array))
- def _str_lower(self):
+ def _str_lower(self) -> Self:
return type(self)(pc.utf8_lower(self._pa_array))
- def _str_upper(self):
+ def _str_upper(self) -> Self:
return type(self)(pc.utf8_upper(self._pa_array))
- def _str_strip(self, to_strip=None):
+ def _str_strip(self, to_strip=None) -> Self:
if to_strip is None:
result = pc.utf8_trim_whitespace(self._pa_array)
else:
result = pc.utf8_trim(self._pa_array, characters=to_strip)
return type(self)(result)
- def _str_lstrip(self, to_strip=None):
+ def _str_lstrip(self, to_strip=None) -> Self:
if to_strip is None:
result = pc.utf8_ltrim_whitespace(self._pa_array)
else:
result = pc.utf8_ltrim(self._pa_array, characters=to_strip)
return type(self)(result)
- def _str_rstrip(self, to_strip=None):
+ def _str_rstrip(self, to_strip=None) -> Self:
if to_strip is None:
result = pc.utf8_rtrim_whitespace(self._pa_array)
else:
@@ -2396,12 +2501,12 @@ def _str_removeprefix(self, prefix: str):
result = self._apply_elementwise(predicate)
return type(self)(pa.chunked_array(result))
- def _str_casefold(self):
+ def _str_casefold(self) -> Self:
predicate = lambda val: val.casefold()
result = self._apply_elementwise(predicate)
return type(self)(pa.chunked_array(result))
- def _str_encode(self, encoding: str, errors: str = "strict"):
+ def _str_encode(self, encoding: str, errors: str = "strict") -> Self:
predicate = lambda val: val.encode(encoding, errors)
result = self._apply_elementwise(predicate)
return type(self)(pa.chunked_array(result))
@@ -2421,7 +2526,7 @@ def _str_extract(self, pat: str, flags: int = 0, expand: bool = True):
else:
return type(self)(pc.struct_field(result, [0]))
- def _str_findall(self, pat: str, flags: int = 0):
+ def _str_findall(self, pat: str, flags: int = 0) -> Self:
regex = re.compile(pat, flags=flags)
predicate = lambda val: regex.findall(val)
result = self._apply_elementwise(predicate)
@@ -2443,22 +2548,22 @@ def _str_get_dummies(self, sep: str = "|"):
result = type(self)(pa.array(list(dummies)))
return result, uniques_sorted.to_pylist()
- def _str_index(self, sub: str, start: int = 0, end: int | None = None):
+ def _str_index(self, sub: str, start: int = 0, end: int | None = None) -> Self:
predicate = lambda val: val.index(sub, start, end)
result = self._apply_elementwise(predicate)
return type(self)(pa.chunked_array(result))
- def _str_rindex(self, sub: str, start: int = 0, end: int | None = None):
+ def _str_rindex(self, sub: str, start: int = 0, end: int | None = None) -> Self:
predicate = lambda val: val.rindex(sub, start, end)
result = self._apply_elementwise(predicate)
return type(self)(pa.chunked_array(result))
- def _str_normalize(self, form: str):
+ def _str_normalize(self, form: str) -> Self:
predicate = lambda val: unicodedata.normalize(form, val)
result = self._apply_elementwise(predicate)
return type(self)(pa.chunked_array(result))
- def _str_rfind(self, sub: str, start: int = 0, end=None):
+ def _str_rfind(self, sub: str, start: int = 0, end=None) -> Self:
predicate = lambda val: val.rfind(sub, start, end)
result = self._apply_elementwise(predicate)
return type(self)(pa.chunked_array(result))
@@ -2469,7 +2574,7 @@ def _str_split(
n: int | None = -1,
expand: bool = False,
regex: bool | None = None,
- ):
+ ) -> Self:
if n in {-1, 0}:
n = None
if pat is None:
@@ -2480,24 +2585,23 @@ def _str_split(
split_func = functools.partial(pc.split_pattern, pattern=pat)
return type(self)(split_func(self._pa_array, max_splits=n))
- def _str_rsplit(self, pat: str | None = None, n: int | None = -1):
+ def _str_rsplit(self, pat: str | None = None, n: int | None = -1) -> Self:
if n in {-1, 0}:
n = None
if pat is None:
return type(self)(
pc.utf8_split_whitespace(self._pa_array, max_splits=n, reverse=True)
)
- else:
- return type(self)(
- pc.split_pattern(self._pa_array, pat, max_splits=n, reverse=True)
- )
+ return type(self)(
+ pc.split_pattern(self._pa_array, pat, max_splits=n, reverse=True)
+ )
- def _str_translate(self, table: dict[int, str]):
+ def _str_translate(self, table: dict[int, str]) -> Self:
predicate = lambda val: val.translate(table)
result = self._apply_elementwise(predicate)
return type(self)(pa.chunked_array(result))
- def _str_wrap(self, width: int, **kwargs):
+ def _str_wrap(self, width: int, **kwargs) -> Self:
kwargs["width"] = width
tw = textwrap.TextWrapper(**kwargs)
predicate = lambda val: "\n".join(tw.wrap(val))
@@ -2505,13 +2609,13 @@ def _str_wrap(self, width: int, **kwargs):
return type(self)(pa.chunked_array(result))
@property
- def _dt_days(self):
+ def _dt_days(self) -> Self:
return type(self)(
pa.array(self._to_timedeltaarray().days, from_pandas=True, type=pa.int32())
)
@property
- def _dt_hours(self):
+ def _dt_hours(self) -> Self:
return type(self)(
pa.array(
[
@@ -2523,7 +2627,7 @@ def _dt_hours(self):
)
@property
- def _dt_minutes(self):
+ def _dt_minutes(self) -> Self:
return type(self)(
pa.array(
[
@@ -2535,7 +2639,7 @@ def _dt_minutes(self):
)
@property
- def _dt_seconds(self):
+ def _dt_seconds(self) -> Self:
return type(self)(
pa.array(
self._to_timedeltaarray().seconds, from_pandas=True, type=pa.int32()
@@ -2543,7 +2647,7 @@ def _dt_seconds(self):
)
@property
- def _dt_milliseconds(self):
+ def _dt_milliseconds(self) -> Self:
return type(self)(
pa.array(
[
@@ -2555,7 +2659,7 @@ def _dt_milliseconds(self):
)
@property
- def _dt_microseconds(self):
+ def _dt_microseconds(self) -> Self:
return type(self)(
pa.array(
self._to_timedeltaarray().microseconds,
@@ -2565,25 +2669,25 @@ def _dt_microseconds(self):
)
@property
- def _dt_nanoseconds(self):
+ def _dt_nanoseconds(self) -> Self:
return type(self)(
pa.array(
self._to_timedeltaarray().nanoseconds, from_pandas=True, type=pa.int32()
)
)
- def _dt_to_pytimedelta(self):
+ def _dt_to_pytimedelta(self) -> np.ndarray:
data = self._pa_array.to_pylist()
if self._dtype.pyarrow_dtype.unit == "ns":
data = [None if ts is None else ts.to_pytimedelta() for ts in data]
return np.array(data, dtype=object)
- def _dt_total_seconds(self):
+ def _dt_total_seconds(self) -> Self:
return type(self)(
pa.array(self._to_timedeltaarray().total_seconds(), from_pandas=True)
)
- def _dt_as_unit(self, unit: str):
+ def _dt_as_unit(self, unit: str) -> Self:
if pa.types.is_date(self.dtype.pyarrow_dtype):
raise NotImplementedError("as_unit not implemented for date types")
pd_array = self._maybe_convert_datelike_array()
@@ -2591,43 +2695,43 @@ def _dt_as_unit(self, unit: str):
return type(self)(pa.array(pd_array.as_unit(unit), from_pandas=True))
@property
- def _dt_year(self):
+ def _dt_year(self) -> Self:
return type(self)(pc.year(self._pa_array))
@property
- def _dt_day(self):
+ def _dt_day(self) -> Self:
return type(self)(pc.day(self._pa_array))
@property
- def _dt_day_of_week(self):
+ def _dt_day_of_week(self) -> Self:
return type(self)(pc.day_of_week(self._pa_array))
_dt_dayofweek = _dt_day_of_week
_dt_weekday = _dt_day_of_week
@property
- def _dt_day_of_year(self):
+ def _dt_day_of_year(self) -> Self:
return type(self)(pc.day_of_year(self._pa_array))
_dt_dayofyear = _dt_day_of_year
@property
- def _dt_hour(self):
+ def _dt_hour(self) -> Self:
return type(self)(pc.hour(self._pa_array))
- def _dt_isocalendar(self):
+ def _dt_isocalendar(self) -> Self:
return type(self)(pc.iso_calendar(self._pa_array))
@property
- def _dt_is_leap_year(self):
+ def _dt_is_leap_year(self) -> Self:
return type(self)(pc.is_leap_year(self._pa_array))
@property
- def _dt_is_month_start(self):
+ def _dt_is_month_start(self) -> Self:
return type(self)(pc.equal(pc.day(self._pa_array), 1))
@property
- def _dt_is_month_end(self):
+ def _dt_is_month_end(self) -> Self:
result = pc.equal(
pc.days_between(
pc.floor_temporal(self._pa_array, unit="day"),
@@ -2638,7 +2742,7 @@ def _dt_is_month_end(self):
return type(self)(result)
@property
- def _dt_is_year_start(self):
+ def _dt_is_year_start(self) -> Self:
return type(self)(
pc.and_(
pc.equal(pc.month(self._pa_array), 1),
@@ -2647,7 +2751,7 @@ def _dt_is_year_start(self):
)
@property
- def _dt_is_year_end(self):
+ def _dt_is_year_end(self) -> Self:
return type(self)(
pc.and_(
pc.equal(pc.month(self._pa_array), 12),
@@ -2656,7 +2760,7 @@ def _dt_is_year_end(self):
)
@property
- def _dt_is_quarter_start(self):
+ def _dt_is_quarter_start(self) -> Self:
result = pc.equal(
pc.floor_temporal(self._pa_array, unit="quarter"),
pc.floor_temporal(self._pa_array, unit="day"),
@@ -2664,7 +2768,7 @@ def _dt_is_quarter_start(self):
return type(self)(result)
@property
- def _dt_is_quarter_end(self):
+ def _dt_is_quarter_end(self) -> Self:
result = pc.equal(
pc.days_between(
pc.floor_temporal(self._pa_array, unit="day"),
@@ -2675,7 +2779,7 @@ def _dt_is_quarter_end(self):
return type(self)(result)
@property
- def _dt_days_in_month(self):
+ def _dt_days_in_month(self) -> Self:
result = pc.days_between(
pc.floor_temporal(self._pa_array, unit="month"),
pc.ceil_temporal(self._pa_array, unit="month"),
@@ -2685,35 +2789,35 @@ def _dt_days_in_month(self):
_dt_daysinmonth = _dt_days_in_month
@property
- def _dt_microsecond(self):
+ def _dt_microsecond(self) -> Self:
return type(self)(pc.microsecond(self._pa_array))
@property
- def _dt_minute(self):
+ def _dt_minute(self) -> Self:
return type(self)(pc.minute(self._pa_array))
@property
- def _dt_month(self):
+ def _dt_month(self) -> Self:
return type(self)(pc.month(self._pa_array))
@property
- def _dt_nanosecond(self):
+ def _dt_nanosecond(self) -> Self:
return type(self)(pc.nanosecond(self._pa_array))
@property
- def _dt_quarter(self):
+ def _dt_quarter(self) -> Self:
return type(self)(pc.quarter(self._pa_array))
@property
- def _dt_second(self):
+ def _dt_second(self) -> Self:
return type(self)(pc.second(self._pa_array))
@property
- def _dt_date(self):
+ def _dt_date(self) -> Self:
return type(self)(self._pa_array.cast(pa.date32()))
@property
- def _dt_time(self):
+ def _dt_time(self) -> Self:
unit = (
self.dtype.pyarrow_dtype.unit
if self.dtype.pyarrow_dtype.unit in {"us", "ns"}
@@ -2729,10 +2833,10 @@ def _dt_tz(self):
def _dt_unit(self):
return self.dtype.pyarrow_dtype.unit
- def _dt_normalize(self):
+ def _dt_normalize(self) -> Self:
return type(self)(pc.floor_temporal(self._pa_array, 1, "day"))
- def _dt_strftime(self, format: str):
+ def _dt_strftime(self, format: str) -> Self:
return type(self)(pc.strftime(self._pa_array, format=format))
def _round_temporally(
@@ -2741,7 +2845,7 @@ def _round_temporally(
freq,
ambiguous: TimeAmbiguous = "raise",
nonexistent: TimeNonexistent = "raise",
- ):
+ ) -> Self:
if ambiguous != "raise":
raise NotImplementedError("ambiguous is not supported.")
if nonexistent != "raise":
@@ -2777,7 +2881,7 @@ def _dt_ceil(
freq,
ambiguous: TimeAmbiguous = "raise",
nonexistent: TimeNonexistent = "raise",
- ):
+ ) -> Self:
return self._round_temporally("ceil", freq, ambiguous, nonexistent)
def _dt_floor(
@@ -2785,7 +2889,7 @@ def _dt_floor(
freq,
ambiguous: TimeAmbiguous = "raise",
nonexistent: TimeNonexistent = "raise",
- ):
+ ) -> Self:
return self._round_temporally("floor", freq, ambiguous, nonexistent)
def _dt_round(
@@ -2793,20 +2897,22 @@ def _dt_round(
freq,
ambiguous: TimeAmbiguous = "raise",
nonexistent: TimeNonexistent = "raise",
- ):
+ ) -> Self:
return self._round_temporally("round", freq, ambiguous, nonexistent)
- def _dt_day_name(self, locale: str | None = None):
+ def _dt_day_name(self, locale: str | None = None) -> Self:
if locale is None:
locale = "C"
return type(self)(pc.strftime(self._pa_array, format="%A", locale=locale))
- def _dt_month_name(self, locale: str | None = None):
+ def _dt_month_name(self, locale: str | None = None) -> Self:
if locale is None:
locale = "C"
return type(self)(pc.strftime(self._pa_array, format="%B", locale=locale))
- def _dt_to_pydatetime(self):
+ def _dt_to_pydatetime(self) -> Series:
+ from pandas import Series
+
if pa.types.is_date(self.dtype.pyarrow_dtype):
raise ValueError(
f"to_pydatetime cannot be called with {self.dtype.pyarrow_dtype} type. "
@@ -2815,14 +2921,14 @@ def _dt_to_pydatetime(self):
data = self._pa_array.to_pylist()
if self._dtype.pyarrow_dtype.unit == "ns":
data = [None if ts is None else ts.to_pydatetime(warn=False) for ts in data]
- return np.array(data, dtype=object)
+ return Series(data, dtype=object)
def _dt_tz_localize(
self,
tz,
ambiguous: TimeAmbiguous = "raise",
nonexistent: TimeNonexistent = "raise",
- ):
+ ) -> Self:
if ambiguous != "raise":
raise NotImplementedError(f"{ambiguous=} is not supported")
nonexistent_pa = {
@@ -2830,7 +2936,8 @@ def _dt_tz_localize(
"shift_backward": "earliest",
"shift_forward": "latest",
}.get(
- nonexistent, None # type: ignore[arg-type]
+ nonexistent, # type: ignore[arg-type]
+ None,
)
if nonexistent_pa is None:
raise NotImplementedError(f"{nonexistent=} is not supported")
@@ -2842,7 +2949,7 @@ def _dt_tz_localize(
)
return type(self)(result)
- def _dt_tz_convert(self, tz):
+ def _dt_tz_convert(self, tz) -> Self:
if self.dtype.pyarrow_dtype.tz is None:
raise TypeError(
"Cannot convert tz-naive timestamps, use tz_localize to localize"
diff --git a/pandas/core/arrays/arrow/extension_types.py b/pandas/core/arrays/arrow/extension_types.py
index d52b60df47adc..2fa5f7a882cc7 100644
--- a/pandas/core/arrays/arrow/extension_types.py
+++ b/pandas/core/arrays/arrow/extension_types.py
@@ -145,7 +145,7 @@ def patch_pyarrow() -> None:
return
class ForbiddenExtensionType(pyarrow.ExtensionType):
- def __arrow_ext_serialize__(self):
+ def __arrow_ext_serialize__(self) -> bytes:
return b""
@classmethod
diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py
index 59c6d911cfaef..76615704f2e33 100644
--- a/pandas/core/arrays/base.py
+++ b/pandas/core/arrays/base.py
@@ -6,6 +6,7 @@
This is an experimental API and subject to breaking changes
without warning.
"""
+
from __future__ import annotations
import operator
@@ -37,7 +38,6 @@
from pandas.util._exceptions import find_stack_level
from pandas.util._validators import (
validate_bool_kwarg,
- validate_fillna_kwargs,
validate_insert_loc,
)
@@ -70,6 +70,7 @@
unique,
)
from pandas.core.array_algos.quantile import quantile_with_mask
+from pandas.core.missing import _fill_limit_area_1d
from pandas.core.sorting import (
nargminmax,
nargsort,
@@ -81,6 +82,7 @@
Sequence,
)
+ from pandas._libs.missing import NAType
from pandas._typing import (
ArrayLike,
AstypeArg,
@@ -266,7 +268,9 @@ class ExtensionArray:
# ------------------------------------------------------------------------
@classmethod
- def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
+ def _from_sequence(
+ cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
+ ) -> Self:
"""
Construct a new ExtensionArray from a sequence of scalars.
@@ -328,8 +332,8 @@ def _from_scalars(cls, scalars, *, dtype: DtypeObj) -> Self:
@classmethod
def _from_sequence_of_strings(
- cls, strings, *, dtype: Dtype | None = None, copy: bool = False
- ):
+ cls, strings, *, dtype: ExtensionDtype, copy: bool = False
+ ) -> Self:
"""
Construct a new ExtensionArray from a sequence of strings.
@@ -338,7 +342,7 @@ def _from_sequence_of_strings(
strings : Sequence
Each element will be an instance of the scalar type for this
array, ``cls.dtype.type``.
- dtype : dtype, optional
+ dtype : ExtensionDtype
Construct for this particular dtype. This should be a Dtype
compatible with the ExtensionArray.
copy : bool, default False
@@ -350,7 +354,9 @@ def _from_sequence_of_strings(
Examples
--------
- >>> pd.arrays.IntegerArray._from_sequence_of_strings(["1", "2", "3"])
+ >>> pd.arrays.IntegerArray._from_sequence_of_strings(
+ ... ["1", "2", "3"], dtype=pd.Int64Dtype()
+ ... )
[1, 2, 3]
Length: 3, dtype: Int64
@@ -376,8 +382,9 @@ def _from_factorized(cls, values, original):
Examples
--------
- >>> interv_arr = pd.arrays.IntervalArray([pd.Interval(0, 1),
- ... pd.Interval(1, 5), pd.Interval(1, 5)])
+ >>> interv_arr = pd.arrays.IntervalArray(
+ ... [pd.Interval(0, 1), pd.Interval(1, 5), pd.Interval(1, 5)]
+ ... )
>>> codes, uniques = pd.factorize(interv_arr)
>>> pd.arrays.IntervalArray._from_factorized(uniques, interv_arr)
@@ -390,12 +397,10 @@ def _from_factorized(cls, values, original):
# Must be a Sequence
# ------------------------------------------------------------------------
@overload
- def __getitem__(self, item: ScalarIndexer) -> Any:
- ...
+ def __getitem__(self, item: ScalarIndexer) -> Any: ...
@overload
- def __getitem__(self, item: SequenceIndexer) -> Self:
- ...
+ def __getitem__(self, item: SequenceIndexer) -> Self: ...
def __getitem__(self, item: PositionalIndexer) -> Self | Any:
"""
@@ -641,16 +646,13 @@ def nbytes(self) -> int:
# ------------------------------------------------------------------------
@overload
- def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray:
- ...
+ def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ...
@overload
- def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray:
- ...
+ def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ...
@overload
- def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike:
- ...
+ def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ...
def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
"""
@@ -681,7 +683,7 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
Casting to another ``ExtensionDtype`` returns an ``ExtensionArray``:
- >>> arr1 = arr.astype('Float64')
+ >>> arr1 = arr.astype("Float64")
>>> arr1
[1.0, 2.0, 3.0]
@@ -691,7 +693,7 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
Otherwise, we will get a Numpy ndarray:
- >>> arr2 = arr.astype('float64')
+ >>> arr2 = arr.astype("float64")
>>> arr2
array([1., 2., 3.])
>>> arr2.dtype
@@ -718,7 +720,10 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
return TimedeltaArray._from_sequence(self, dtype=dtype, copy=copy)
- return np.array(self, dtype=dtype, copy=copy)
+ if not copy:
+ return np.asarray(self, dtype=dtype)
+ else:
+ return np.array(self, dtype=dtype, copy=copy)
def isna(self) -> np.ndarray | ExtensionArraySupportsAnyAll:
"""
@@ -814,7 +819,7 @@ def argsort(
na_position : {'first', 'last'}, default 'last'
If ``'first'``, put ``NaN`` values at the beginning.
If ``'last'``, put ``NaN`` values at the end.
- *args, **kwargs:
+ **kwargs
Passed through to :func:`numpy.argsort`.
Returns
@@ -935,15 +940,16 @@ def interpolate(
Examples
--------
>>> arr = pd.arrays.NumpyExtensionArray(np.array([0, 1, np.nan, 3]))
- >>> arr.interpolate(method="linear",
- ... limit=3,
- ... limit_direction="forward",
- ... index=pd.Index([1, 2, 3, 4]),
- ... fill_value=1,
- ... copy=False,
- ... axis=0,
- ... limit_area="inside"
- ... )
+ >>> arr.interpolate(
+ ... method="linear",
+ ... limit=3,
+ ... limit_direction="forward",
+ ... index=pd.Index([1, 2, 3, 4]),
+ ... fill_value=1,
+ ... copy=False,
+ ... axis=0,
+ ... limit_area="inside",
+ ... )
[0.0, 1.0, 2.0, 3.0]
Length: 4, dtype: float64
@@ -954,7 +960,12 @@ def interpolate(
)
def _pad_or_backfill(
- self, *, method: FillnaOptions, limit: int | None = None, copy: bool = True
+ self,
+ *,
+ method: FillnaOptions,
+ limit: int | None = None,
+ limit_area: Literal["inside", "outside"] | None = None,
+ copy: bool = True,
) -> Self:
"""
Pad or backfill values, used by Series/DataFrame ffill and bfill.
@@ -995,25 +1006,6 @@ def _pad_or_backfill(
[, 2, 2, 3, , ]
Length: 6, dtype: Int64
"""
-
- # If a 3rd-party EA has implemented this functionality in fillna,
- # we warn that they need to implement _pad_or_backfill instead.
- if (
- type(self).fillna is not ExtensionArray.fillna
- and type(self)._pad_or_backfill is ExtensionArray._pad_or_backfill
- ):
- # Check for _pad_or_backfill here allows us to call
- # super()._pad_or_backfill without getting this warning
- warnings.warn(
- "ExtensionArray.fillna 'method' keyword is deprecated. "
- "In a future version. arr._pad_or_backfill will be called "
- "instead. 3rd-party ExtensionArray authors need to implement "
- "_pad_or_backfill.",
- DeprecationWarning,
- stacklevel=find_stack_level(),
- )
- return self.fillna(method=method, limit=limit)
-
mask = self.isna()
if mask.any():
@@ -1021,6 +1013,8 @@ def _pad_or_backfill(
meth = missing.clean_fill_method(method)
npmask = np.asarray(mask)
+ if limit_area is not None and not npmask.all():
+ _fill_limit_area_1d(npmask, limit_area)
if meth == "pad":
indexer = libalgos.get_fill_indexer(npmask, limit=limit)
return self.take(indexer, allow_fill=True)
@@ -1037,8 +1031,7 @@ def _pad_or_backfill(
def fillna(
self,
- value: object | ArrayLike | None = None,
- method: FillnaOptions | None = None,
+ value: object | ArrayLike,
limit: int | None = None,
copy: bool = True,
) -> Self:
@@ -1051,14 +1044,6 @@ def fillna(
If a scalar value is passed it is used to fill all missing values.
Alternatively, an array-like "value" can be given. It's expected
that the array-like have the same length as 'self'.
- method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
- Method to use for filling holes in reindexed Series:
-
- * pad / ffill: propagate last valid observation forward to next valid.
- * backfill / bfill: use NEXT valid observation to fill gap.
-
- .. deprecated:: 2.1.0
-
limit : int, default None
If method is specified, this is the maximum number of consecutive
NaN values to forward/backward fill. In other words, if there is
@@ -1066,9 +1051,6 @@ def fillna(
be partially filled. If method is not specified, this is the
maximum number of entries along the entire axis where NaNs will be
filled.
-
- .. deprecated:: 2.1.0
-
copy : bool, default True
Whether to make a copy of the data before filling. If False, then
the original should be modified and no new memory should be allocated.
@@ -1090,42 +1072,22 @@ def fillna(
[0, 0, 2, 3, 0, 0]
Length: 6, dtype: Int64
"""
- if method is not None:
- warnings.warn(
- f"The 'method' keyword in {type(self).__name__}.fillna is "
- "deprecated and will be removed in a future version.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- value, method = validate_fillna_kwargs(value, method)
-
mask = self.isna()
# error: Argument 2 to "check_value_size" has incompatible type
# "ExtensionArray"; expected "ndarray"
value = missing.check_value_size(
- value, mask, len(self) # type: ignore[arg-type]
+ value,
+ mask, # type: ignore[arg-type]
+ len(self),
)
if mask.any():
- if method is not None:
- meth = missing.clean_fill_method(method)
-
- npmask = np.asarray(mask)
- if meth == "pad":
- indexer = libalgos.get_fill_indexer(npmask, limit=limit)
- return self.take(indexer, allow_fill=True)
- else:
- # i.e. meth == "backfill"
- indexer = libalgos.get_fill_indexer(npmask[::-1], limit=limit)[::-1]
- return self[::-1].take(indexer, allow_fill=True)
+ # fill with value
+ if not copy:
+ new_values = self[:]
else:
- # fill with value
- if not copy:
- new_values = self[:]
- else:
- new_values = self.copy()
- new_values[mask] = value
+ new_values = self.copy()
+ new_values[mask] = value
else:
if not copy:
new_values = self[:]
@@ -1448,8 +1410,10 @@ def factorize(
Examples
--------
- >>> idx1 = pd.PeriodIndex(["2014-01", "2014-01", "2014-02", "2014-02",
- ... "2014-03", "2014-03"], freq="M")
+ >>> idx1 = pd.PeriodIndex(
+ ... ["2014-01", "2014-01", "2014-02", "2014-02", "2014-03", "2014-03"],
+ ... freq="M",
+ ... )
>>> arr, idx = idx1.factorize()
>>> arr
array([0, 0, 1, 1, 2, 2])
@@ -1473,9 +1437,7 @@ def factorize(
uniques_ea = self._from_factorized(uniques, self)
return codes, uniques_ea
- _extension_array_shared_docs[
- "repeat"
- ] = """
+ _extension_array_shared_docs["repeat"] = """
Repeat elements of a %(klass)s.
Returns a new %(klass)s where each element of the current %(klass)s
@@ -1610,10 +1572,9 @@ def take(self, indices, allow_fill=False, fill_value=None):
# type for the array, to the physical storage type for
# the data, before passing to take.
- result = take(data, indices, fill_value=fill_value,
- allow_fill=allow_fill)
+ result = take(data, indices, fill_value=fill_value, allow_fill=allow_fill)
return self._from_sequence(result, dtype=self.dtype)
- """
+ """ # noqa: E501
# Implementer note: The `fill_value` parameter should be a user-facing
# value, an instance of self.dtype.type. When passed `fill_value=None`,
# the default of `self.dtype.na_value` should be used.
@@ -1750,7 +1711,7 @@ def _formatter(self, boxed: bool = False) -> Callable[[Any], str | None]:
--------
>>> class MyExtensionArray(pd.arrays.NumpyExtensionArray):
... def _formatter(self, boxed=False):
- ... return lambda x: '*' + str(x) + '*' if boxed else repr(x) + '*'
+ ... return lambda x: "*" + str(x) + "*" if boxed else repr(x) + "*"
>>> MyExtensionArray(np.array([1, 2, 3, 4]))
[1*, 2*, 3*, 4*]
@@ -1764,7 +1725,7 @@ def _formatter(self, boxed: bool = False) -> Callable[[Any], str | None]:
# Reshaping
# ------------------------------------------------------------------------
- def transpose(self, *axes: int) -> ExtensionArray:
+ def transpose(self, *axes: int) -> Self:
"""
Return a transposed view on this array.
@@ -1785,10 +1746,10 @@ def transpose(self, *axes: int) -> ExtensionArray:
return self[:]
@property
- def T(self) -> ExtensionArray:
+ def T(self) -> Self:
return self.transpose()
- def ravel(self, order: Literal["C", "F", "A", "K"] | None = "C") -> ExtensionArray:
+ def ravel(self, order: Literal["C", "F", "A", "K"] | None = "C") -> Self:
"""
Return a flattened view on this array.
@@ -1885,7 +1846,7 @@ def _accumulate(
Examples
--------
>>> arr = pd.array([1, 2, 3])
- >>> arr._accumulate(name='cumsum')
+ >>> arr._accumulate(name="cumsum")
[1, 3, 6]
Length: 3, dtype: Int64
@@ -1990,10 +1951,9 @@ def _hash_pandas_object(
Examples
--------
- >>> pd.array([1, 2])._hash_pandas_object(encoding='utf-8',
- ... hash_key="1000000000000000",
- ... categorize=False
- ... )
+ >>> pd.array([1, 2])._hash_pandas_object(
+ ... encoding="utf-8", hash_key="1000000000000000", categorize=False
+ ... )
array([ 6238072747940578789, 15839785061582574730], dtype=uint64)
"""
from pandas.core.util.hashing import hash_array
@@ -2027,8 +1987,9 @@ def _explode(self) -> tuple[Self, npt.NDArray[np.uint64]]:
Examples
--------
>>> import pyarrow as pa
- >>> a = pd.array([[1, 2, 3], [4], [5, 6]],
- ... dtype=pd.ArrowDtype(pa.list_(pa.int64())))
+ >>> a = pd.array(
+ ... [[1, 2, 3], [4], [5, 6]], dtype=pd.ArrowDtype(pa.list_(pa.int64()))
+ ... )
>>> a._explode()
(
[1, 2, 3, 4, 5, 6]
@@ -2185,7 +2146,7 @@ def _rank(
raise NotImplementedError
return rank(
- self._values_for_argsort(),
+ self,
axis=axis,
method=method,
na_option=na_option,
@@ -2385,10 +2346,22 @@ def _groupby_op(
class ExtensionArraySupportsAnyAll(ExtensionArray):
- def any(self, *, skipna: bool = True) -> bool:
+ @overload
+ def any(self, *, skipna: Literal[True] = ...) -> bool: ...
+
+ @overload
+ def any(self, *, skipna: bool) -> bool | NAType: ...
+
+ def any(self, *, skipna: bool = True) -> bool | NAType:
raise AbstractMethodError(self)
- def all(self, *, skipna: bool = True) -> bool:
+ @overload
+ def all(self, *, skipna: Literal[True] = ...) -> bool: ...
+
+ @overload
+ def all(self, *, skipna: bool) -> bool | NAType: ...
+
+ def all(self, *, skipna: bool = True) -> bool | NAType:
raise AbstractMethodError(self)
diff --git a/pandas/core/arrays/boolean.py b/pandas/core/arrays/boolean.py
index 04e6f0a0bcdde..813b10eef5e4b 100644
--- a/pandas/core/arrays/boolean.py
+++ b/pandas/core/arrays/boolean.py
@@ -29,13 +29,14 @@
import pyarrow
from pandas._typing import (
- Dtype,
DtypeObj,
Self,
npt,
type_t,
)
+ from pandas.core.dtypes.dtypes import ExtensionDtype
+
@register_extension_dtype
class BooleanDtype(BaseMaskedDtype):
@@ -55,6 +56,10 @@ class BooleanDtype(BaseMaskedDtype):
-------
None
+ See Also
+ --------
+ StringDtype : Extension dtype for string data.
+
Examples
--------
>>> pd.BooleanDtype()
@@ -324,7 +329,7 @@ def _from_sequence_of_strings(
cls,
strings: list[str],
*,
- dtype: Dtype | None = None,
+ dtype: ExtensionDtype,
copy: bool = False,
true_values: list[str] | None = None,
false_values: list[str] | None = None,
diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py
index 8a88227ad54a3..416331a260e9f 100644
--- a/pandas/core/arrays/categorical.py
+++ b/pandas/core/arrays/categorical.py
@@ -44,7 +44,9 @@
pandas_dtype,
)
from pandas.core.dtypes.dtypes import (
+ ArrowDtype,
CategoricalDtype,
+ CategoricalDtypeType,
ExtensionDtype,
)
from pandas.core.dtypes.generic import (
@@ -241,7 +243,9 @@ def contains(cat, key, container) -> bool:
return any(loc_ in container for loc_ in loc)
-class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMixin):
+# error: Definition of "delete/ravel/T/repeat/copy" in base class "NDArrayBacked"
+# is incompatible with definition in base class "ExtensionArray"
+class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMixin): # type: ignore[misc]
"""
Represent a categorical variable in classic R / S-plus fashion.
@@ -272,6 +276,11 @@ class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMi
provided).
dtype : CategoricalDtype
An instance of ``CategoricalDtype`` to use for this categorical.
+ fastpath : bool
+ The 'fastpath' keyword in Categorical is deprecated and will be
+ removed in a future version. Use Categorical.from_codes instead.
+ copy : bool, default True
+ Whether to copy if the codes are unchanged.
Attributes
----------
@@ -289,6 +298,15 @@ class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMi
Methods
-------
from_codes
+ as_ordered
+ as_unordered
+ set_categories
+ rename_categories
+ reorder_categories
+ add_categories
+ remove_categories
+ remove_unused_categories
+ map
__array__
Raises
@@ -316,7 +334,7 @@ class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMi
[1, 2, 3, 1, 2, 3]
Categories (3, int64): [1, 2, 3]
- >>> pd.Categorical(['a', 'b', 'c', 'a', 'b', 'c'])
+ >>> pd.Categorical(["a", "b", "c", "a", "b", "c"])
['a', 'b', 'c', 'a', 'b', 'c']
Categories (3, object): ['a', 'b', 'c']
@@ -336,8 +354,9 @@ class Categorical(NDArrayBackedExtensionArray, PandasObject, ObjectStringArrayMi
Ordered `Categoricals` can be sorted according to the custom order
of the categories and can have a min and max value.
- >>> c = pd.Categorical(['a', 'b', 'c', 'a', 'b', 'c'], ordered=True,
- ... categories=['c', 'b', 'a'])
+ >>> c = pd.Categorical(
+ ... ["a", "b", "c", "a", "b", "c"], ordered=True, categories=["c", "b", "a"]
+ ... )
>>> c
['a', 'b', 'c', 'a', 'b', 'c']
Categories (3, object): ['c' < 'b' < 'a']
@@ -412,6 +431,10 @@ def __init__(
if isinstance(vdtype, CategoricalDtype):
if dtype.categories is None:
dtype = CategoricalDtype(values.categories, dtype.ordered)
+ elif isinstance(values, range):
+ from pandas.core.indexes.range import RangeIndex
+
+ values = RangeIndex(values)
elif not isinstance(values, (ABCIndex, ABCSeries, ExtensionArray)):
values = com.convert_to_list_like(values)
if isinstance(values, list) and len(values) == 0:
@@ -443,24 +466,32 @@ def __init__(
values = arr
if dtype.categories is None:
- if not isinstance(values, ABCIndex):
- # in particular RangeIndex xref test_index_equal_range_categories
- values = sanitize_array(values, None)
- try:
- codes, categories = factorize(values, sort=True)
- except TypeError as err:
- codes, categories = factorize(values, sort=False)
- if dtype.ordered:
- # raise, as we don't have a sortable data structure and so
- # the user should give us one by specifying categories
- raise TypeError(
- "'values' is not ordered, please "
- "explicitly specify the categories order "
- "by passing in a categories argument."
- ) from err
-
- # we're inferring from values
- dtype = CategoricalDtype(categories, dtype.ordered)
+ if isinstance(values.dtype, ArrowDtype) and issubclass(
+ values.dtype.type, CategoricalDtypeType
+ ):
+ arr = values._pa_array.combine_chunks()
+ categories = arr.dictionary.to_pandas(types_mapper=ArrowDtype)
+ codes = arr.indices.to_numpy()
+ dtype = CategoricalDtype(categories, values.dtype.pyarrow_dtype.ordered)
+ else:
+ if not isinstance(values, ABCIndex):
+ # in particular RangeIndex xref test_index_equal_range_categories
+ values = sanitize_array(values, None)
+ try:
+ codes, categories = factorize(values, sort=True)
+ except TypeError as err:
+ codes, categories = factorize(values, sort=False)
+ if dtype.ordered:
+ # raise, as we don't have a sortable data structure and so
+ # the user should give us one by specifying categories
+ raise TypeError(
+ "'values' is not ordered, please "
+ "explicitly specify the categories order "
+ "by passing in a categories argument."
+ ) from err
+
+ # we're inferring from values
+ dtype = CategoricalDtype(categories, dtype.ordered)
elif isinstance(values.dtype, CategoricalDtype):
old_codes = extract_array(values)._codes
@@ -488,7 +519,7 @@ def dtype(self) -> CategoricalDtype:
Examples
--------
- >>> cat = pd.Categorical(['a', 'b'], ordered=True)
+ >>> cat = pd.Categorical(["a", "b"], ordered=True)
>>> cat
['a', 'b']
Categories (2, object): ['a' < 'b']
@@ -527,16 +558,13 @@ def _from_scalars(cls, scalars, *, dtype: DtypeObj) -> Self:
return res
@overload
- def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray:
- ...
+ def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ...
@overload
- def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray:
- ...
+ def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ...
@overload
- def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike:
- ...
+ def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ...
def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
"""
@@ -551,6 +579,7 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
object is returned.
"""
dtype = pandas_dtype(dtype)
+ result: Categorical | np.ndarray
if self.dtype is dtype:
result = self.copy() if copy else self
@@ -587,9 +616,9 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
except (
TypeError, # downstream error msg for CategoricalIndex is misleading
ValueError,
- ):
+ ) as err:
msg = f"Cannot cast {self.categories.dtype} dtype to {dtype}"
- raise ValueError(msg)
+ raise ValueError(msg) from err
result = take_nd(
new_cats, ensure_platform_int(self._codes), fill_value=fill_value
@@ -597,19 +626,6 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
return result
- def to_list(self):
- """
- Alias for tolist.
- """
- # GH#51254
- warnings.warn(
- "Categorical.to_list is deprecated and will be removed in a future "
- "version. Use obj.tolist() instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self.tolist()
-
@classmethod
def _from_inferred_categories(
cls, inferred_categories, inferred_codes, dtype, true_values=None
@@ -727,7 +743,7 @@ def from_codes(
Examples
--------
- >>> dtype = pd.CategoricalDtype(['a', 'b'], ordered=True)
+ >>> dtype = pd.CategoricalDtype(["a", "b"], ordered=True)
>>> pd.Categorical.from_codes(codes=[0, 1, 0, 1], dtype=dtype)
['a', 'b', 'a', 'b']
Categories (2, object): ['a' < 'b']
@@ -782,28 +798,28 @@ def categories(self) -> Index:
--------
For :class:`pandas.Series`:
- >>> ser = pd.Series(['a', 'b', 'c', 'a'], dtype='category')
+ >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
>>> ser.cat.categories
Index(['a', 'b', 'c'], dtype='object')
- >>> raw_cat = pd.Categorical(['a', 'b', 'c', 'a'], categories=['b', 'c', 'd'])
+ >>> raw_cat = pd.Categorical(["a", "b", "c", "a"], categories=["b", "c", "d"])
>>> ser = pd.Series(raw_cat)
>>> ser.cat.categories
Index(['b', 'c', 'd'], dtype='object')
For :class:`pandas.Categorical`:
- >>> cat = pd.Categorical(['a', 'b'], ordered=True)
+ >>> cat = pd.Categorical(["a", "b"], ordered=True)
>>> cat.categories
Index(['a', 'b'], dtype='object')
For :class:`pandas.CategoricalIndex`:
- >>> ci = pd.CategoricalIndex(['a', 'c', 'b', 'a', 'c', 'b'])
+ >>> ci = pd.CategoricalIndex(["a", "c", "b", "a", "c", "b"])
>>> ci.categories
Index(['a', 'b', 'c'], dtype='object')
- >>> ci = pd.CategoricalIndex(['a', 'c'], categories=['c', 'b', 'a'])
+ >>> ci = pd.CategoricalIndex(["a", "c"], categories=["c", "b", "a"])
>>> ci.categories
Index(['c', 'b', 'a'], dtype='object')
"""
@@ -818,32 +834,32 @@ def ordered(self) -> Ordered:
--------
For :class:`pandas.Series`:
- >>> ser = pd.Series(['a', 'b', 'c', 'a'], dtype='category')
+ >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
>>> ser.cat.ordered
False
- >>> raw_cat = pd.Categorical(['a', 'b', 'c', 'a'], ordered=True)
+ >>> raw_cat = pd.Categorical(["a", "b", "c", "a"], ordered=True)
>>> ser = pd.Series(raw_cat)
>>> ser.cat.ordered
True
For :class:`pandas.Categorical`:
- >>> cat = pd.Categorical(['a', 'b'], ordered=True)
+ >>> cat = pd.Categorical(["a", "b"], ordered=True)
>>> cat.ordered
True
- >>> cat = pd.Categorical(['a', 'b'], ordered=False)
+ >>> cat = pd.Categorical(["a", "b"], ordered=False)
>>> cat.ordered
False
For :class:`pandas.CategoricalIndex`:
- >>> ci = pd.CategoricalIndex(['a', 'b'], ordered=True)
+ >>> ci = pd.CategoricalIndex(["a", "b"], ordered=True)
>>> ci.ordered
True
- >>> ci = pd.CategoricalIndex(['a', 'b'], ordered=False)
+ >>> ci = pd.CategoricalIndex(["a", "b"], ordered=False)
>>> ci.ordered
False
"""
@@ -869,17 +885,17 @@ def codes(self) -> np.ndarray:
--------
For :class:`pandas.Categorical`:
- >>> cat = pd.Categorical(['a', 'b'], ordered=True)
+ >>> cat = pd.Categorical(["a", "b"], ordered=True)
>>> cat.codes
array([0, 1], dtype=int8)
For :class:`pandas.CategoricalIndex`:
- >>> ci = pd.CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'])
+ >>> ci = pd.CategoricalIndex(["a", "b", "c", "a", "b", "c"])
>>> ci.codes
array([0, 1, 2, 0, 1, 2], dtype=int8)
- >>> ci = pd.CategoricalIndex(['a', 'c'], categories=['c', 'b', 'a'])
+ >>> ci = pd.CategoricalIndex(["a", "c"], categories=["c", "b", "a"])
>>> ci.codes
array([2, 0], dtype=int8)
"""
@@ -898,12 +914,12 @@ def _set_categories(self, categories, fastpath: bool = False) -> None:
Examples
--------
- >>> c = pd.Categorical(['a', 'b'])
+ >>> c = pd.Categorical(["a", "b"])
>>> c
['a', 'b']
Categories (2, object): ['a', 'b']
- >>> c._set_categories(pd.Index(['a', 'c']))
+ >>> c._set_categories(pd.Index(["a", "c"]))
>>> c
['a', 'c']
Categories (2, object): ['a', 'c']
@@ -963,11 +979,15 @@ def as_ordered(self) -> Self:
Categorical
Ordered Categorical.
+ See Also
+ --------
+ as_unordered : Set the Categorical to be unordered.
+
Examples
--------
For :class:`pandas.Series`:
- >>> ser = pd.Series(['a', 'b', 'c', 'a'], dtype='category')
+ >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
>>> ser.cat.ordered
False
>>> ser = ser.cat.as_ordered()
@@ -976,7 +996,7 @@ def as_ordered(self) -> Self:
For :class:`pandas.CategoricalIndex`:
- >>> ci = pd.CategoricalIndex(['a', 'b', 'c', 'a'])
+ >>> ci = pd.CategoricalIndex(["a", "b", "c", "a"])
>>> ci.ordered
False
>>> ci = ci.as_ordered()
@@ -994,11 +1014,15 @@ def as_unordered(self) -> Self:
Categorical
Unordered Categorical.
+ See Also
+ --------
+ as_ordered : Set the Categorical to be ordered.
+
Examples
--------
For :class:`pandas.Series`:
- >>> raw_cat = pd.Categorical(['a', 'b', 'c', 'a'], ordered=True)
+ >>> raw_cat = pd.Categorical(["a", "b", "c", "a"], ordered=True)
>>> ser = pd.Series(raw_cat)
>>> ser.cat.ordered
True
@@ -1008,7 +1032,7 @@ def as_unordered(self) -> Self:
For :class:`pandas.CategoricalIndex`:
- >>> ci = pd.CategoricalIndex(['a', 'b', 'c', 'a'], ordered=True)
+ >>> ci = pd.CategoricalIndex(["a", "b", "c", "a"], ordered=True)
>>> ci.ordered
True
>>> ci = ci.as_unordered()
@@ -1017,7 +1041,9 @@ def as_unordered(self) -> Self:
"""
return self.set_ordered(False)
- def set_categories(self, new_categories, ordered=None, rename: bool = False):
+ def set_categories(
+ self, new_categories, ordered=None, rename: bool = False
+ ) -> Self:
"""
Set the categories to the specified new categories.
@@ -1034,14 +1060,14 @@ def set_categories(self, new_categories, ordered=None, rename: bool = False):
On the other hand this methods does not do checks (e.g., whether the
old categories are included in the new categories on a reorder), which
can result in surprising changes, for example when using special string
- dtypes, which does not considers a S1 string equal to a single char
+ dtypes, which do not consider a S1 string equal to a single char
python string.
Parameters
----------
new_categories : Index-like
The categories in new order.
- ordered : bool, default False
+ ordered : bool, default None
Whether or not the categorical is treated as a ordered categorical.
If not given, do not change the ordered information.
rename : bool, default False
@@ -1050,7 +1076,8 @@ def set_categories(self, new_categories, ordered=None, rename: bool = False):
Returns
-------
- Categorical with reordered categories.
+ Categorical
+ New categories to be used, with optional ordering changes.
Raises
------
@@ -1069,8 +1096,9 @@ def set_categories(self, new_categories, ordered=None, rename: bool = False):
--------
For :class:`pandas.Series`:
- >>> raw_cat = pd.Categorical(['a', 'b', 'c', 'A'],
- ... categories=['a', 'b', 'c'], ordered=True)
+ >>> raw_cat = pd.Categorical(
+ ... ["a", "b", "c", "A"], categories=["a", "b", "c"], ordered=True
+ ... )
>>> ser = pd.Series(raw_cat)
>>> ser
0 a
@@ -1080,7 +1108,7 @@ def set_categories(self, new_categories, ordered=None, rename: bool = False):
dtype: category
Categories (3, object): ['a' < 'b' < 'c']
- >>> ser.cat.set_categories(['A', 'B', 'C'], rename=True)
+ >>> ser.cat.set_categories(["A", "B", "C"], rename=True)
0 A
1 B
2 C
@@ -1090,16 +1118,17 @@ def set_categories(self, new_categories, ordered=None, rename: bool = False):
For :class:`pandas.CategoricalIndex`:
- >>> ci = pd.CategoricalIndex(['a', 'b', 'c', 'A'],
- ... categories=['a', 'b', 'c'], ordered=True)
+ >>> ci = pd.CategoricalIndex(
+ ... ["a", "b", "c", "A"], categories=["a", "b", "c"], ordered=True
+ ... )
>>> ci
CategoricalIndex(['a', 'b', 'c', nan], categories=['a', 'b', 'c'],
ordered=True, dtype='category')
- >>> ci.set_categories(['A', 'b', 'c'])
+ >>> ci.set_categories(["A", "b", "c"])
CategoricalIndex([nan, 'b', 'c', nan], categories=['A', 'b', 'c'],
ordered=True, dtype='category')
- >>> ci.set_categories(['A', 'b', 'c'], rename=True)
+ >>> ci.set_categories(["A", "b", "c"], rename=True)
CategoricalIndex(['A', 'b', 'c', nan], categories=['A', 'b', 'c'],
ordered=True, dtype='category')
"""
@@ -1165,7 +1194,7 @@ def rename_categories(self, new_categories) -> Self:
Examples
--------
- >>> c = pd.Categorical(['a', 'a', 'b'])
+ >>> c = pd.Categorical(["a", "a", "b"])
>>> c.rename_categories([0, 1])
[0, 0, 1]
Categories (2, int64): [0, 1]
@@ -1173,7 +1202,7 @@ def rename_categories(self, new_categories) -> Self:
For dict-like ``new_categories``, extra keys are ignored and
categories not in the dictionary are passed through
- >>> c.rename_categories({'a': 'A', 'c': 'C'})
+ >>> c.rename_categories({"a": "A", "c": "C"})
['A', 'A', 'b']
Categories (2, object): ['A', 'b']
@@ -1233,8 +1262,8 @@ def reorder_categories(self, new_categories, ordered=None) -> Self:
--------
For :class:`pandas.Series`:
- >>> ser = pd.Series(['a', 'b', 'c', 'a'], dtype='category')
- >>> ser = ser.cat.reorder_categories(['c', 'b', 'a'], ordered=True)
+ >>> ser = pd.Series(["a", "b", "c", "a"], dtype="category")
+ >>> ser = ser.cat.reorder_categories(["c", "b", "a"], ordered=True)
>>> ser
0 a
1 b
@@ -1253,11 +1282,11 @@ def reorder_categories(self, new_categories, ordered=None) -> Self:
For :class:`pandas.CategoricalIndex`:
- >>> ci = pd.CategoricalIndex(['a', 'b', 'c', 'a'])
+ >>> ci = pd.CategoricalIndex(["a", "b", "c", "a"])
>>> ci
CategoricalIndex(['a', 'b', 'c', 'a'], categories=['a', 'b', 'c'],
ordered=False, dtype='category')
- >>> ci.reorder_categories(['c', 'b', 'a'], ordered=True)
+ >>> ci.reorder_categories(["c", "b", "a"], ordered=True)
CategoricalIndex(['a', 'b', 'c', 'a'], categories=['c', 'b', 'a'],
ordered=True, dtype='category')
"""
@@ -1303,12 +1332,12 @@ def add_categories(self, new_categories) -> Self:
Examples
--------
- >>> c = pd.Categorical(['c', 'b', 'c'])
+ >>> c = pd.Categorical(["c", "b", "c"])
>>> c
['c', 'b', 'c']
Categories (2, object): ['b', 'c']
- >>> c.add_categories(['d', 'a'])
+ >>> c.add_categories(["d", "a"])
['c', 'b', 'c']
Categories (4, object): ['b', 'c', 'd', 'a']
"""
@@ -1371,12 +1400,12 @@ def remove_categories(self, removals) -> Self:
Examples
--------
- >>> c = pd.Categorical(['a', 'c', 'b', 'c', 'd'])
+ >>> c = pd.Categorical(["a", "c", "b", "c", "d"])
>>> c
['a', 'c', 'b', 'c', 'd']
Categories (4, object): ['a', 'b', 'c', 'd']
- >>> c.remove_categories(['d', 'a'])
+ >>> c.remove_categories(["d", "a"])
[NaN, 'c', 'b', 'c', NaN]
Categories (2, object): ['b', 'c']
"""
@@ -1418,13 +1447,13 @@ def remove_unused_categories(self) -> Self:
Examples
--------
- >>> c = pd.Categorical(['a', 'c', 'b', 'c', 'd'])
+ >>> c = pd.Categorical(["a", "c", "b", "c", "d"])
>>> c
['a', 'c', 'b', 'c', 'd']
Categories (4, object): ['a', 'b', 'c', 'd']
- >>> c[2] = 'a'
- >>> c[4] = 'c'
+ >>> c[2] = "a"
+ >>> c[4] = "c"
>>> c
['a', 'c', 'a', 'c', 'c']
Categories (4, object): ['a', 'b', 'c', 'd']
@@ -1498,37 +1527,37 @@ def map(
Examples
--------
- >>> cat = pd.Categorical(['a', 'b', 'c'])
+ >>> cat = pd.Categorical(["a", "b", "c"])
>>> cat
['a', 'b', 'c']
Categories (3, object): ['a', 'b', 'c']
>>> cat.map(lambda x: x.upper(), na_action=None)
['A', 'B', 'C']
Categories (3, object): ['A', 'B', 'C']
- >>> cat.map({'a': 'first', 'b': 'second', 'c': 'third'}, na_action=None)
+ >>> cat.map({"a": "first", "b": "second", "c": "third"}, na_action=None)
['first', 'second', 'third']
Categories (3, object): ['first', 'second', 'third']
If the mapping is one-to-one the ordering of the categories is
preserved:
- >>> cat = pd.Categorical(['a', 'b', 'c'], ordered=True)
+ >>> cat = pd.Categorical(["a", "b", "c"], ordered=True)
>>> cat
['a', 'b', 'c']
Categories (3, object): ['a' < 'b' < 'c']
- >>> cat.map({'a': 3, 'b': 2, 'c': 1}, na_action=None)
+ >>> cat.map({"a": 3, "b": 2, "c": 1}, na_action=None)
[3, 2, 1]
Categories (3, int64): [3 < 2 < 1]
If the mapping is not one-to-one an :class:`~pandas.Index` is returned:
- >>> cat.map({'a': 'first', 'b': 'second', 'c': 'first'}, na_action=None)
+ >>> cat.map({"a": "first", "b": "second", "c": "first"}, na_action=None)
Index(['first', 'second', 'first'], dtype='object')
If a `dict` is used, all unmapped categories are mapped to `NaN` and
the result is an :class:`~pandas.Index`:
- >>> cat.map({'a': 'first', 'b': 'second'}, na_action=None)
+ >>> cat.map({"a": "first", "b": "second"}, na_action=None)
Index(['first', 'second', nan], dtype='object')
"""
if na_action is lib.no_default:
@@ -1626,10 +1655,20 @@ def _validate_codes_for_dtype(cls, codes, *, dtype: CategoricalDtype) -> np.ndar
# -------------------------------------------------------------
@ravel_compat
- def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+ def __array__(
+ self, dtype: NpDtype | None = None, copy: bool | None = None
+ ) -> np.ndarray:
"""
The numpy array interface.
+ Parameters
+ ----------
+ dtype : np.dtype or None
+ Specifies the the dtype for the array.
+
+ copy : bool or None, optional
+ Unused.
+
Returns
-------
numpy.array
@@ -1640,7 +1679,7 @@ def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
Examples
--------
- >>> cat = pd.Categorical(['a', 'b'], ordered=True)
+ >>> cat = pd.Categorical(["a", "b"], ordered=True)
The following calls ``cat.__array__``
@@ -1806,10 +1845,14 @@ def value_counts(self, dropna: bool = True) -> Series:
ix = np.append(ix, -1)
ix = coerce_indexer_dtype(ix, self.dtype.categories)
- ix = self._from_backing_data(ix)
+ ix_categorical = self._from_backing_data(ix)
return Series(
- count, index=CategoricalIndex(ix), dtype="int64", name="count", copy=False
+ count,
+ index=CategoricalIndex(ix_categorical),
+ dtype="int64",
+ name="count",
+ copy=False,
)
# error: Argument 2 of "_empty" is incompatible with supertype
@@ -1870,7 +1913,7 @@ def check_for_ordered(self, op) -> None:
def argsort(
self, *, ascending: bool = True, kind: SortKind = "quicksort", **kwargs
- ):
+ ) -> npt.NDArray[np.intp]:
"""
Return the indices that would sort the Categorical.
@@ -1904,12 +1947,12 @@ def argsort(
Examples
--------
- >>> pd.Categorical(['b', 'b', 'a', 'c']).argsort()
+ >>> pd.Categorical(["b", "b", "a", "c"]).argsort()
array([2, 0, 1, 3])
- >>> cat = pd.Categorical(['b', 'b', 'a', 'c'],
- ... categories=['c', 'b', 'a'],
- ... ordered=True)
+ >>> cat = pd.Categorical(
+ ... ["b", "b", "a", "c"], categories=["c", "b", "a"], ordered=True
+ ... )
>>> cat.argsort()
array([3, 0, 1, 2])
@@ -1928,14 +1971,12 @@ def sort_values(
inplace: Literal[False] = ...,
ascending: bool = ...,
na_position: str = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def sort_values(
self, *, inplace: Literal[True], ascending: bool = ..., na_position: str = ...
- ) -> None:
- ...
+ ) -> None: ...
def sort_values(
self,
@@ -2003,16 +2044,16 @@ def sort_values(
>>> c.sort_values(ascending=False)
[5, 2, 2, NaN, NaN]
Categories (2, int64): [2, 5]
- >>> c.sort_values(na_position='first')
+ >>> c.sort_values(na_position="first")
[NaN, NaN, 2, 2, 5]
Categories (2, int64): [2, 5]
- >>> c.sort_values(ascending=False, na_position='first')
+ >>> c.sort_values(ascending=False, na_position="first")
[NaN, NaN, 5, 2, 2]
Categories (2, int64): [2, 5]
"""
inplace = validate_bool_kwarg(inplace, "inplace")
if na_position not in ["last", "first"]:
- raise ValueError(f"invalid na_position: {repr(na_position)}")
+ raise ValueError(f"invalid na_position: {na_position!r}")
sorted_idx = nargsort(self, ascending=ascending, na_position=na_position)
@@ -2320,7 +2361,7 @@ def _reverse_indexer(self) -> dict[Hashable, npt.NDArray[np.intp]]:
Examples
--------
- >>> c = pd.Categorical(list('aabca'))
+ >>> c = pd.Categorical(list("aabca"))
>>> c
['a', 'a', 'b', 'c', 'a']
Categories (3, object): ['a', 'b', 'c']
@@ -2514,7 +2555,9 @@ def _concat_same_type(cls, to_concat: Sequence[Self], axis: AxisInt = 0) -> Self
result = res_flat.reshape(len(first), -1, order="F")
return result
- result = union_categoricals(to_concat)
+ # error: Incompatible types in assignment (expression has type "Categorical",
+ # variable has type "Self")
+ result = union_categoricals(to_concat) # type: ignore[assignment]
return result
# ------------------------------------------------------------------
@@ -2566,7 +2609,7 @@ def describe(self) -> DataFrame:
from pandas import Index
from pandas.core.reshape.concat import concat
- result = concat([counts, freqs], axis=1)
+ result = concat([counts, freqs], ignore_index=True, axis=1)
result.columns = Index(["counts", "freqs"])
result.index.name = "categories"
@@ -2602,15 +2645,14 @@ def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]:
Examples
--------
- >>> s = pd.Categorical(['lama', 'cow', 'lama', 'beetle', 'lama',
- ... 'hippo'])
- >>> s.isin(['cow', 'lama'])
+ >>> s = pd.Categorical(["llama", "cow", "llama", "beetle", "llama", "hippo"])
+ >>> s.isin(["cow", "llama"])
array([ True, True, True, False, True, False])
- Passing a single string as ``s.isin('lama')`` will raise an error. Use
+ Passing a single string as ``s.isin('llama')`` will raise an error. Use
a list of one element instead:
- >>> s.isin(['lama'])
+ >>> s.isin(["llama"])
array([ True, False, True, False, True, False])
"""
null_mask = np.asarray(isna(values))
@@ -2618,7 +2660,13 @@ def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]:
code_values = code_values[null_mask | (code_values >= 0)]
return algorithms.isin(self.codes, code_values)
- def _replace(self, *, to_replace, value, inplace: bool = False):
+ @overload
+ def _replace(self, *, to_replace, value, inplace: Literal[False] = ...) -> Self: ...
+
+ @overload
+ def _replace(self, *, to_replace, value, inplace: Literal[True]) -> None: ...
+
+ def _replace(self, *, to_replace, value, inplace: bool = False) -> Self | None:
from pandas import Index
orig_dtype = self.dtype
@@ -2646,11 +2694,11 @@ def _replace(self, *, to_replace, value, inplace: bool = False):
new_categories = ser.take(locs)
new_categories = new_categories.drop_duplicates(keep="first")
- new_categories = Index(new_categories)
+ index_categories = Index(new_categories)
new_codes = recode_for_categories(
- cat._codes, all_values, new_categories, copy=False
+ cat._codes, all_values, index_categories, copy=False
)
- new_dtype = CategoricalDtype(new_categories, ordered=self.dtype.ordered)
+ new_dtype = CategoricalDtype(index_categories, ordered=self.dtype.ordered)
NDArrayBacked.__init__(cat, new_codes, new_dtype)
if new_dtype != orig_dtype:
@@ -2666,6 +2714,7 @@ def _replace(self, *, to_replace, value, inplace: bool = False):
)
if not inplace:
return cat
+ return None
# ------------------------------------------------------------------------
# String methods interface
@@ -2901,8 +2950,8 @@ def _delegate_property_get(self, name: str):
# error: Signature of "_delegate_property_set" incompatible with supertype
# "PandasDelegate"
- def _delegate_property_set(self, name: str, new_values): # type: ignore[override]
- return setattr(self._parent, name, new_values)
+ def _delegate_property_set(self, name: str, new_values) -> None: # type: ignore[override]
+ setattr(self._parent, name, new_values)
@property
def codes(self) -> Series:
@@ -2968,8 +3017,8 @@ def recode_for_categories(
Examples
--------
- >>> old_cat = pd.Index(['b', 'a', 'c'])
- >>> new_cat = pd.Index(['a', 'b'])
+ >>> old_cat = pd.Index(["b", "a", "c"])
+ >>> new_cat = pd.Index(["a", "b"])
>>> codes = np.array([0, 1, 1, 2])
>>> recode_for_categories(codes, old_cat, new_cat)
array([ 1, 0, 0, -1], dtype=int8)
diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py
index 6ca74c4c05bc6..52cb175ca79a2 100644
--- a/pandas/core/arrays/datetimelike.py
+++ b/pandas/core/arrays/datetimelike.py
@@ -20,11 +20,12 @@
import numpy as np
+from pandas._config.config import get_option
+
from pandas._libs import (
algos,
lib,
)
-from pandas._libs.arrays import NDArrayBacked
from pandas._libs.tslibs import (
BaseOffset,
IncompatibleFrequency,
@@ -270,7 +271,7 @@ def _unbox_scalar(
Examples
--------
- >>> arr = pd.array(np.array(['1970-01-01'], 'datetime64[ns]'))
+ >>> arr = pd.array(np.array(["1970-01-01"], "datetime64[ns]"))
>>> arr._unbox_scalar(arr[0])
numpy.datetime64('1970-01-01T00:00:00.000000000')
"""
@@ -344,29 +345,29 @@ def _format_native_types(
"""
raise AbstractMethodError(self)
- def _formatter(self, boxed: bool = False):
+ def _formatter(self, boxed: bool = False) -> Callable[[object], str]:
# TODO: Remove Datetime & DatetimeTZ formatters.
return "'{}'".format
# ----------------------------------------------------------------
# Array-Like / EA-Interface Methods
- def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+ def __array__(
+ self, dtype: NpDtype | None = None, copy: bool | None = None
+ ) -> np.ndarray:
# used for Timedelta/DatetimeArray, overwritten by PeriodArray
if is_object_dtype(dtype):
return np.array(list(self), dtype=object)
return self._ndarray
@overload
- def __getitem__(self, item: ScalarIndexer) -> DTScalarOrNaT:
- ...
+ def __getitem__(self, key: ScalarIndexer) -> DTScalarOrNaT: ...
@overload
def __getitem__(
self,
- item: SequenceIndexer | PositionalIndexerTuple,
- ) -> Self:
- ...
+ key: SequenceIndexer | PositionalIndexerTuple,
+ ) -> Self: ...
def __getitem__(self, key: PositionalIndexer2D) -> Self | DTScalarOrNaT:
"""
@@ -494,20 +495,16 @@ def astype(self, dtype, copy: bool = True):
return np.asarray(self, dtype=dtype)
@overload
- def view(self) -> Self:
- ...
+ def view(self) -> Self: ...
@overload
- def view(self, dtype: Literal["M8[ns]"]) -> DatetimeArray:
- ...
+ def view(self, dtype: Literal["M8[ns]"]) -> DatetimeArray: ...
@overload
- def view(self, dtype: Literal["m8[ns]"]) -> TimedeltaArray:
- ...
+ def view(self, dtype: Literal["m8[ns]"]) -> TimedeltaArray: ...
@overload
- def view(self, dtype: Dtype | None = ...) -> ArrayLike:
- ...
+ def view(self, dtype: Dtype | None = ...) -> ArrayLike: ...
# pylint: disable-next=useless-parent-delegation
def view(self, dtype: Dtype | None = None) -> ArrayLike:
@@ -524,9 +521,9 @@ def _validate_comparison_value(self, other):
try:
# GH#18435 strings get a pass from tzawareness compat
other = self._scalar_from_string(other)
- except (ValueError, IncompatibleFrequency):
+ except (ValueError, IncompatibleFrequency) as err:
# failed to parse as Timestamp/Timedelta/Period
- raise InvalidComparison(other)
+ raise InvalidComparison(other) from err
if isinstance(other, self._recognized_scalars) or other is NaT:
other = self._scalar_type(other)
@@ -649,7 +646,7 @@ def _validation_error_message(self, value, allow_listlike: bool = False) -> str:
def _validate_listlike(self, value, allow_object: bool = False):
if isinstance(value, type(self)):
- if self.dtype.kind in "mM" and not allow_object:
+ if self.dtype.kind in "mM" and not allow_object and self.unit != value.unit: # type: ignore[attr-defined]
# error: "DatetimeLikeArrayMixin" has no attribute "as_unit"
value = value.as_unit(self.unit, round_ok=False) # type: ignore[attr-defined]
return value
@@ -664,11 +661,11 @@ def _validate_listlike(self, value, allow_object: bool = False):
if lib.infer_dtype(value) in self._infer_matches:
try:
value = type(self)._from_sequence(value)
- except (ValueError, TypeError):
+ except (ValueError, TypeError) as err:
if allow_object:
return value
msg = self._validation_error_message(value, True)
- raise TypeError(msg)
+ raise TypeError(msg) from err
# Do type inference if necessary up front (after unpacking
# NumpyExtensionArray)
@@ -808,9 +805,8 @@ def isin(self, values: ArrayLike) -> npt.NDArray[np.bool_]:
if self.dtype.kind in "mM":
self = cast("DatetimeArray | TimedeltaArray", self)
- # error: Item "ExtensionArray" of "ExtensionArray | ndarray[Any, Any]"
- # has no attribute "as_unit"
- values = values.as_unit(self.unit) # type: ignore[union-attr]
+ # error: "DatetimeLikeArrayMixin" has no attribute "as_unit"
+ values = values.as_unit(self.unit) # type: ignore[attr-defined]
try:
# error: Argument 1 to "_check_compatible_with" of "DatetimeLikeArrayMixin"
@@ -890,8 +886,9 @@ def freqstr(self) -> str | None:
The frequency can be inferred if there are more than 2 points:
- >>> idx = pd.DatetimeIndex(["2018-01-01", "2018-01-03", "2018-01-05"],
- ... freq="infer")
+ >>> idx = pd.DatetimeIndex(
+ ... ["2018-01-01", "2018-01-03", "2018-01-05"], freq="infer"
+ ... )
>>> idx.freqstr
'2D'
@@ -1209,7 +1206,7 @@ def _add_timedeltalike_scalar(self, other):
self, other = self._ensure_matching_resos(other)
return self._add_timedeltalike(other)
- def _add_timedelta_arraylike(self, other: TimedeltaArray):
+ def _add_timedelta_arraylike(self, other: TimedeltaArray) -> Self:
"""
Add a delta of a TimedeltaIndex
@@ -1222,30 +1219,28 @@ def _add_timedelta_arraylike(self, other: TimedeltaArray):
if len(self) != len(other):
raise ValueError("cannot add indices of unequal length")
- self = cast("DatetimeArray | TimedeltaArray", self)
-
- self, other = self._ensure_matching_resos(other)
+ self, other = cast(
+ "DatetimeArray | TimedeltaArray", self
+ )._ensure_matching_resos(other)
return self._add_timedeltalike(other)
@final
- def _add_timedeltalike(self, other: Timedelta | TimedeltaArray):
- self = cast("DatetimeArray | TimedeltaArray", self)
-
+ def _add_timedeltalike(self, other: Timedelta | TimedeltaArray) -> Self:
other_i8, o_mask = self._get_i8_values_and_mask(other)
new_values = add_overflowsafe(self.asi8, np.asarray(other_i8, dtype="i8"))
res_values = new_values.view(self._ndarray.dtype)
new_freq = self._get_arithmetic_result_freq(other)
- # error: Argument "dtype" to "_simple_new" of "DatetimeArray" has
- # incompatible type "Union[dtype[datetime64], DatetimeTZDtype,
- # dtype[timedelta64]]"; expected "Union[dtype[datetime64], DatetimeTZDtype]"
+ # error: Unexpected keyword argument "freq" for "_simple_new" of "NDArrayBacked"
return type(self)._simple_new(
- res_values, dtype=self.dtype, freq=new_freq # type: ignore[arg-type]
+ res_values,
+ dtype=self.dtype,
+ freq=new_freq, # type: ignore[call-arg]
)
@final
- def _add_nat(self):
+ def _add_nat(self) -> Self:
"""
Add pd.NaT to self
"""
@@ -1253,22 +1248,21 @@ def _add_nat(self):
raise TypeError(
f"Cannot add {type(self).__name__} and {type(NaT).__name__}"
)
- self = cast("TimedeltaArray | DatetimeArray", self)
# GH#19124 pd.NaT is treated like a timedelta for both timedelta
# and datetime dtypes
result = np.empty(self.shape, dtype=np.int64)
result.fill(iNaT)
result = result.view(self._ndarray.dtype) # preserve reso
- # error: Argument "dtype" to "_simple_new" of "DatetimeArray" has
- # incompatible type "Union[dtype[timedelta64], dtype[datetime64],
- # DatetimeTZDtype]"; expected "Union[dtype[datetime64], DatetimeTZDtype]"
+ # error: Unexpected keyword argument "freq" for "_simple_new" of "NDArrayBacked"
return type(self)._simple_new(
- result, dtype=self.dtype, freq=None # type: ignore[arg-type]
+ result,
+ dtype=self.dtype,
+ freq=None, # type: ignore[call-arg]
)
@final
- def _sub_nat(self):
+ def _sub_nat(self) -> np.ndarray:
"""
Subtract pd.NaT from self
"""
@@ -1313,7 +1307,7 @@ def _sub_periodlike(self, other: Period | PeriodArray) -> npt.NDArray[np.object_
return new_data
@final
- def _addsub_object_array(self, other: npt.NDArray[np.object_], op):
+ def _addsub_object_array(self, other: npt.NDArray[np.object_], op) -> np.ndarray:
"""
Add or subtract array-like of DateOffset objects
@@ -1335,12 +1329,13 @@ def _addsub_object_array(self, other: npt.NDArray[np.object_], op):
# If both 1D then broadcasting is unambiguous
return op(self, other[0])
- warnings.warn(
- "Adding/subtracting object-dtype array to "
- f"{type(self).__name__} not vectorized.",
- PerformanceWarning,
- stacklevel=find_stack_level(),
- )
+ if get_option("performance_warnings"):
+ warnings.warn(
+ "Adding/subtracting object-dtype array to "
+ f"{type(self).__name__} not vectorized.",
+ PerformanceWarning,
+ stacklevel=find_stack_level(),
+ )
# Caller is responsible for broadcasting if necessary
assert self.shape == other.shape, (self.shape, other.shape)
@@ -1364,7 +1359,7 @@ def __add__(self, other):
# scalar others
if other is NaT:
- result = self._add_nat()
+ result: np.ndarray | DatetimeLikeArrayMixin = self._add_nat()
elif isinstance(other, (Tick, timedelta, np.timedelta64)):
result = self._add_timedeltalike_scalar(other)
elif isinstance(other, BaseOffset):
@@ -1424,7 +1419,7 @@ def __sub__(self, other):
# scalar others
if other is NaT:
- result = self._sub_nat()
+ result: np.ndarray | DatetimeLikeArrayMixin = self._sub_nat()
elif isinstance(other, (Tick, timedelta, np.timedelta64)):
result = self._add_timedeltalike_scalar(-other)
elif isinstance(other, BaseOffset):
@@ -1581,6 +1576,7 @@ def mean(self, *, skipna: bool = True, axis: AxisInt | None = 0):
skipna : bool, default True
Whether to ignore any NaT elements.
axis : int, optional, default 0
+ Axis for the function to be applied on.
Returns
-------
@@ -1600,7 +1596,7 @@ def mean(self, *, skipna: bool = True, axis: AxisInt | None = 0):
--------
For :class:`pandas.DatetimeIndex`:
- >>> idx = pd.date_range('2001-01-01 00:00', periods=3)
+ >>> idx = pd.date_range("2001-01-01 00:00", periods=3)
>>> idx
DatetimeIndex(['2001-01-01', '2001-01-02', '2001-01-03'],
dtype='datetime64[ns]', freq='D')
@@ -1609,7 +1605,7 @@ def mean(self, *, skipna: bool = True, axis: AxisInt | None = 0):
For :class:`pandas.TimedeltaIndex`:
- >>> tdelta_idx = pd.to_timedelta([1, 2, 3], unit='D')
+ >>> tdelta_idx = pd.to_timedelta([1, 2, 3], unit="D")
>>> tdelta_idx
TimedeltaIndex(['1 days', '2 days', '3 days'],
dtype='timedelta64[ns]', freq=None)
@@ -1665,16 +1661,8 @@ def _groupby_op(
dtype = self.dtype
if dtype.kind == "M":
# Adding/multiplying datetimes is not valid
- if how in ["sum", "prod", "cumsum", "cumprod", "var", "skew"]:
- raise TypeError(f"datetime64 type does not support {how} operations")
- if how in ["any", "all"]:
- # GH#34479
- warnings.warn(
- f"'{how}' with datetime64 dtypes is deprecated and will raise in a "
- f"future version. Use (obj != pd.Timestamp(0)).{how}() instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
+ if how in ["any", "all", "sum", "prod", "cumsum", "cumprod", "var", "skew"]:
+ raise TypeError(f"datetime64 type does not support operation: '{how}'")
elif isinstance(dtype, PeriodDtype):
# Adding/multiplying Periods is not valid
@@ -1779,9 +1767,8 @@ def strftime(self, date_format: str) -> npt.NDArray[np.object_]:
Examples
--------
- >>> rng = pd.date_range(pd.Timestamp("2018-03-10 09:00"),
- ... periods=3, freq='s')
- >>> rng.strftime('%%B %%d, %%Y, %%r')
+ >>> rng = pd.date_range(pd.Timestamp("2018-03-10 09:00"), periods=3, freq="s")
+ >>> rng.strftime("%%B %%d, %%Y, %%r")
Index(['March 10, 2018, 09:00:00 AM', 'March 10, 2018, 09:00:01 AM',
'March 10, 2018, 09:00:02 AM'],
dtype='object')
@@ -1940,100 +1927,6 @@ class TimelikeOps(DatetimeLikeArrayMixin):
Common ops for TimedeltaIndex/DatetimeIndex, but not PeriodIndex.
"""
- _default_dtype: np.dtype
-
- def __init__(
- self, values, dtype=None, freq=lib.no_default, copy: bool = False
- ) -> None:
- warnings.warn(
- # GH#55623
- f"{type(self).__name__}.__init__ is deprecated and will be "
- "removed in a future version. Use pd.array instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- if dtype is not None:
- dtype = pandas_dtype(dtype)
-
- values = extract_array(values, extract_numpy=True)
- if isinstance(values, IntegerArray):
- values = values.to_numpy("int64", na_value=iNaT)
-
- inferred_freq = getattr(values, "_freq", None)
- explicit_none = freq is None
- freq = freq if freq is not lib.no_default else None
-
- if isinstance(values, type(self)):
- if explicit_none:
- # don't inherit from values
- pass
- elif freq is None:
- freq = values.freq
- elif freq and values.freq:
- freq = to_offset(freq)
- freq = _validate_inferred_freq(freq, values.freq)
-
- if dtype is not None and dtype != values.dtype:
- # TODO: we only have tests for this for DTA, not TDA (2022-07-01)
- raise TypeError(
- f"dtype={dtype} does not match data dtype {values.dtype}"
- )
-
- dtype = values.dtype
- values = values._ndarray
-
- elif dtype is None:
- if isinstance(values, np.ndarray) and values.dtype.kind in "Mm":
- dtype = values.dtype
- else:
- dtype = self._default_dtype
- if isinstance(values, np.ndarray) and values.dtype == "i8":
- values = values.view(dtype)
-
- if not isinstance(values, np.ndarray):
- raise ValueError(
- f"Unexpected type '{type(values).__name__}'. 'values' must be a "
- f"{type(self).__name__}, ndarray, or Series or Index "
- "containing one of those."
- )
- if values.ndim not in [1, 2]:
- raise ValueError("Only 1-dimensional input arrays are supported.")
-
- if values.dtype == "i8":
- # for compat with datetime/timedelta/period shared methods,
- # we can sometimes get here with int64 values. These represent
- # nanosecond UTC (or tz-naive) unix timestamps
- if dtype is None:
- dtype = self._default_dtype
- values = values.view(self._default_dtype)
- elif lib.is_np_dtype(dtype, "mM"):
- values = values.view(dtype)
- elif isinstance(dtype, DatetimeTZDtype):
- kind = self._default_dtype.kind
- new_dtype = f"{kind}8[{dtype.unit}]"
- values = values.view(new_dtype)
-
- dtype = self._validate_dtype(values, dtype)
-
- if freq == "infer":
- raise ValueError(
- f"Frequency inference not allowed in {type(self).__name__}.__init__. "
- "Use 'pd.array()' instead."
- )
-
- if copy:
- values = values.copy()
- if freq:
- freq = to_offset(freq)
- if values.dtype.kind == "m" and not isinstance(freq, Tick):
- raise TypeError("TimedeltaArray/Index freq must be a Tick")
-
- NDArrayBacked.__init__(self, values=values, dtype=dtype)
- self._freq = freq
-
- if inferred_freq is None and freq is not None:
- type(self)._validate_frequency(self, freq)
-
@classmethod
def _validate_dtype(cls, values, dtype):
raise AbstractMethodError(cls)
@@ -2042,6 +1935,29 @@ def _validate_dtype(cls, values, dtype):
def freq(self):
"""
Return the frequency object if it is set, otherwise None.
+
+ To learn more about the frequency strings, please see
+ :ref:`this link`.
+
+ See Also
+ --------
+ DatetimeIndex.freq : Return the frequency object if it is set, otherwise None.
+ PeriodIndex.freq : Return the frequency object if it is set, otherwise None.
+
+ Examples
+ --------
+ >>> datetimeindex = pd.date_range(
+ ... "2022-02-22 02:22:22", periods=10, tz="America/Chicago", freq="h"
+ ... )
+ >>> datetimeindex
+ DatetimeIndex(['2022-02-22 02:22:22-06:00', '2022-02-22 03:22:22-06:00',
+ '2022-02-22 04:22:22-06:00', '2022-02-22 05:22:22-06:00',
+ '2022-02-22 06:22:22-06:00', '2022-02-22 07:22:22-06:00',
+ '2022-02-22 08:22:22-06:00', '2022-02-22 09:22:22-06:00',
+ '2022-02-22 10:22:22-06:00', '2022-02-22 11:22:22-06:00'],
+ dtype='datetime64[ns, America/Chicago]', freq='h')
+ >>> datetimeindex.freq
+
"""
return self._freq
@@ -2155,6 +2071,47 @@ def unit(self) -> str:
return dtype_to_unit(self.dtype) # type: ignore[arg-type]
def as_unit(self, unit: str, round_ok: bool = True) -> Self:
+ """
+ Convert to a dtype with the given unit resolution.
+
+ The limits of timestamp representation depend on the chosen resolution.
+ Different resolutions can be converted to each other through as_unit.
+
+ Parameters
+ ----------
+ unit : {'s', 'ms', 'us', 'ns'}
+ round_ok : bool, default True
+ If False and the conversion requires rounding, raise ValueError.
+
+ Returns
+ -------
+ same type as self
+ Converted to the specified unit.
+
+ See Also
+ --------
+ Timestamp.as_unit : Convert to the given unit.
+
+ Examples
+ --------
+ For :class:`pandas.DatetimeIndex`:
+
+ >>> idx = pd.DatetimeIndex(["2020-01-02 01:02:03.004005006"])
+ >>> idx
+ DatetimeIndex(['2020-01-02 01:02:03.004005006'],
+ dtype='datetime64[ns]', freq=None)
+ >>> idx.as_unit("s")
+ DatetimeIndex(['2020-01-02 01:02:03'], dtype='datetime64[s]', freq=None)
+
+ For :class:`pandas.TimedeltaIndex`:
+
+ >>> tdelta_idx = pd.to_timedelta(["1 day 3 min 2 us 42 ns"])
+ >>> tdelta_idx
+ TimedeltaIndex(['1 days 00:03:00.000002042'],
+ dtype='timedelta64[ns]', freq=None)
+ >>> tdelta_idx.as_unit("s")
+ TimedeltaIndex(['1 days 00:03:00'], dtype='timedelta64[s]', freq=None)
+ """
if unit not in ["s", "ms", "us", "ns"]:
raise ValueError("Supported units are 's', 'ms', 'us', 'ns'")
@@ -2170,7 +2127,9 @@ def as_unit(self, unit: str, round_ok: bool = True) -> Self:
# error: Unexpected keyword argument "freq" for "_simple_new" of
# "NDArrayBacked" [call-arg]
return type(self)._simple_new(
- new_values, dtype=new_dtype, freq=self.freq # type: ignore[call-arg]
+ new_values,
+ dtype=new_dtype,
+ freq=self.freq, # type: ignore[call-arg]
)
# TODO: annotate other as DatetimeArray | TimedeltaArray | Timestamp | Timedelta
@@ -2250,11 +2209,11 @@ def ceil(
# Reductions
def any(self, *, axis: AxisInt | None = None, skipna: bool = True) -> bool:
- # GH#34479 the nanops call will issue a FutureWarning for non-td64 dtype
+ # GH#34479 the nanops call will raise a TypeError for non-td64 dtype
return nanops.nanany(self._ndarray, axis=axis, skipna=skipna, mask=self.isna())
def all(self, *, axis: AxisInt | None = None, skipna: bool = True) -> bool:
- # GH#34479 the nanops call will issue a FutureWarning for non-td64 dtype
+ # GH#34479 the nanops call will raise a TypeError for non-td64 dtype
return nanops.nanall(self._ndarray, axis=axis, skipna=skipna, mask=self.isna())
@@ -2309,11 +2268,12 @@ def factorize(
):
if self.freq is not None:
# We must be unique, so can short-circuit (and retain freq)
- codes = np.arange(len(self), dtype=np.intp)
- uniques = self.copy() # TODO: copy or view?
if sort and self.freq.n < 0:
- codes = codes[::-1]
- uniques = uniques[::-1]
+ codes = np.arange(len(self) - 1, -1, -1, dtype=np.intp)
+ uniques = self[::-1]
+ else:
+ codes = np.arange(len(self), dtype=np.intp)
+ uniques = self.copy() # TODO: copy or view?
return codes, uniques
if sort:
@@ -2460,13 +2420,11 @@ def ensure_arraylike_for_datetimelike(
@overload
-def validate_periods(periods: None) -> None:
- ...
+def validate_periods(periods: None) -> None: ...
@overload
-def validate_periods(periods: int | float) -> int:
- ...
+def validate_periods(periods: int | float) -> int: ...
def validate_periods(periods: int | float | None) -> int | None:
diff --git a/pandas/core/arrays/datetimes.py b/pandas/core/arrays/datetimes.py
index 6b7ddc4a72957..d446407ec3d01 100644
--- a/pandas/core/arrays/datetimes.py
+++ b/pandas/core/arrays/datetimes.py
@@ -7,6 +7,7 @@
)
from typing import (
TYPE_CHECKING,
+ TypeVar,
cast,
overload,
)
@@ -14,6 +15,8 @@
import numpy as np
+from pandas._config.config import get_option
+
from pandas._libs import (
lib,
tslib,
@@ -39,10 +42,7 @@
tz_convert_from_utc,
tzconversion,
)
-from pandas._libs.tslibs.dtypes import (
- abbrev_to_npy_unit,
- freq_to_period_freqstr,
-)
+from pandas._libs.tslibs.dtypes import abbrev_to_npy_unit
from pandas.errors import PerformanceWarning
from pandas.util._exceptions import find_stack_level
from pandas.util._validators import validate_inclusive
@@ -73,7 +73,10 @@
)
if TYPE_CHECKING:
- from collections.abc import Iterator
+ from collections.abc import (
+ Generator,
+ Iterator,
+ )
from pandas._typing import (
ArrayLike,
@@ -86,21 +89,25 @@
npt,
)
- from pandas import DataFrame
+ from pandas import (
+ DataFrame,
+ Timedelta,
+ )
from pandas.core.arrays import PeriodArray
+ _TimestampNoneT1 = TypeVar("_TimestampNoneT1", Timestamp, None)
+ _TimestampNoneT2 = TypeVar("_TimestampNoneT2", Timestamp, None)
+
_ITER_CHUNKSIZE = 10_000
@overload
-def tz_to_dtype(tz: tzinfo, unit: str = ...) -> DatetimeTZDtype:
- ...
+def tz_to_dtype(tz: tzinfo, unit: str = ...) -> DatetimeTZDtype: ...
@overload
-def tz_to_dtype(tz: None, unit: str = ...) -> np.dtype[np.datetime64]:
- ...
+def tz_to_dtype(tz: None, unit: str = ...) -> np.dtype[np.datetime64]: ...
def tz_to_dtype(
@@ -179,7 +186,7 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps): # type: ignore[misc]
Parameters
----------
- values : Series, Index, DatetimeArray, ndarray
+ data : Series, Index, DatetimeArray, ndarray
The datetime data.
For DatetimeArray `values` (or a Series or Index boxing one),
@@ -203,7 +210,8 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps): # type: ignore[misc]
Examples
--------
>>> pd.arrays.DatetimeArray._from_sequence(
- ... pd.DatetimeIndex(['2023-01-01', '2023-01-02'], freq='D'))
+ ... pd.DatetimeIndex(["2023-01-01", "2023-01-02"], freq="D")
+ ... )
['2023-01-01 00:00:00', '2023-01-02 00:00:00']
Length: 2, dtype: datetime64[ns]
@@ -279,7 +287,6 @@ def _scalar_type(self) -> type[Timestamp]:
_dtype: np.dtype[np.datetime64] | DatetimeTZDtype
_freq: BaseOffset | None = None
- _default_dtype = DT64NS_DTYPE # used in TimeLikeOps.__init__
@classmethod
def _from_scalars(cls, scalars, *, dtype: DtypeObj) -> Self:
@@ -326,7 +333,7 @@ def _simple_new( # type: ignore[override]
return result
@classmethod
- def _from_sequence(cls, scalars, *, dtype=None, copy: bool = False):
+ def _from_sequence(cls, scalars, *, dtype=None, copy: bool = False) -> Self:
return cls._from_sequence_not_strict(scalars, dtype=dtype, copy=copy)
@classmethod
@@ -601,8 +608,9 @@ def tz(self) -> tzinfo | None:
For DatetimeIndex:
- >>> idx = pd.DatetimeIndex(["1/1/2020 10:00:00+00:00",
- ... "2/1/2020 11:00:00+00:00"])
+ >>> idx = pd.DatetimeIndex(
+ ... ["1/1/2020 10:00:00+00:00", "2/1/2020 11:00:00+00:00"]
+ ... )
>>> idx.tz
datetime.timezone.utc
"""
@@ -638,12 +646,12 @@ def _resolution_obj(self) -> Resolution:
# ----------------------------------------------------------------
# Array-Like / EA-Interface Methods
- def __array__(self, dtype=None) -> np.ndarray:
+ def __array__(self, dtype=None, copy=None) -> np.ndarray:
if dtype is None and self.tz:
# The default for tz-aware is object, to preserve tz info
dtype = object
- return super().__array__(dtype=dtype)
+ return super().__array__(dtype=dtype, copy=copy)
def __iter__(self) -> Iterator:
"""
@@ -758,17 +766,6 @@ def _format_native_types(
# -----------------------------------------------------------------
# Comparison Methods
- def _has_same_tz(self, other) -> bool:
- # vzone shouldn't be None if value is non-datetime like
- if isinstance(other, np.datetime64):
- # convert to Timestamp as np.datetime64 doesn't have tz attr
- other = Timestamp(other)
-
- if not hasattr(other, "tzinfo"):
- return False
- other_tz = other.tzinfo
- return timezones.tz_compare(self.tzinfo, other_tz)
-
def _assert_tzawareness_compat(self, other) -> None:
# adapted from _Timestamp._assert_tzawareness_compat
other_tz = getattr(other, "tzinfo", None)
@@ -809,11 +806,13 @@ def _add_offset(self, offset: BaseOffset) -> Self:
# "dtype[Any] | type[Any] | _SupportsDType[dtype[Any]]"
res_values = res_values.view(values.dtype) # type: ignore[arg-type]
except NotImplementedError:
- warnings.warn(
- "Non-vectorized DateOffset being applied to Series or DatetimeIndex.",
- PerformanceWarning,
- stacklevel=find_stack_level(),
- )
+ if get_option("performance_warnings"):
+ warnings.warn(
+ "Non-vectorized DateOffset being applied to Series or "
+ "DatetimeIndex.",
+ PerformanceWarning,
+ stacklevel=find_stack_level(),
+ )
res_values = self.astype("O") + offset
# TODO(GH#55564): as_unit will be unnecessary
result = type(self)._from_sequence(res_values).as_unit(self.unit)
@@ -878,8 +877,9 @@ def tz_convert(self, tz) -> Self:
With the `tz` parameter, we can change the DatetimeIndex
to other time zones:
- >>> dti = pd.date_range(start='2014-08-01 09:00',
- ... freq='h', periods=3, tz='Europe/Berlin')
+ >>> dti = pd.date_range(
+ ... start="2014-08-01 09:00", freq="h", periods=3, tz="Europe/Berlin"
+ ... )
>>> dti
DatetimeIndex(['2014-08-01 09:00:00+02:00',
@@ -887,7 +887,7 @@ def tz_convert(self, tz) -> Self:
'2014-08-01 11:00:00+02:00'],
dtype='datetime64[ns, Europe/Berlin]', freq='h')
- >>> dti.tz_convert('US/Central')
+ >>> dti.tz_convert("US/Central")
DatetimeIndex(['2014-08-01 02:00:00-05:00',
'2014-08-01 03:00:00-05:00',
'2014-08-01 04:00:00-05:00'],
@@ -896,8 +896,9 @@ def tz_convert(self, tz) -> Self:
With the ``tz=None``, we can remove the timezone (after converting
to UTC if necessary):
- >>> dti = pd.date_range(start='2014-08-01 09:00', freq='h',
- ... periods=3, tz='Europe/Berlin')
+ >>> dti = pd.date_range(
+ ... start="2014-08-01 09:00", freq="h", periods=3, tz="Europe/Berlin"
+ ... )
>>> dti
DatetimeIndex(['2014-08-01 09:00:00+02:00',
@@ -1121,7 +1122,7 @@ def to_pydatetime(self) -> npt.NDArray[np.object_]:
Examples
--------
- >>> idx = pd.date_range('2018-02-27', periods=3)
+ >>> idx = pd.date_range("2018-02-27", periods=3)
>>> idx.to_pydatetime()
array([datetime.datetime(2018, 2, 27, 0, 0),
datetime.datetime(2018, 2, 28, 0, 0),
@@ -1154,8 +1155,9 @@ def normalize(self) -> Self:
Examples
--------
- >>> idx = pd.date_range(start='2014-08-01 10:00', freq='h',
- ... periods=3, tz='Asia/Calcutta')
+ >>> idx = pd.date_range(
+ ... start="2014-08-01 10:00", freq="h", periods=3, tz="Asia/Calcutta"
+ ... )
>>> idx
DatetimeIndex(['2014-08-01 10:00:00+05:30',
'2014-08-01 11:00:00+05:30',
@@ -1205,10 +1207,16 @@ def to_period(self, freq=None) -> PeriodArray:
Examples
--------
- >>> df = pd.DataFrame({"y": [1, 2, 3]},
- ... index=pd.to_datetime(["2000-03-31 00:00:00",
- ... "2000-05-31 00:00:00",
- ... "2000-08-31 00:00:00"]))
+ >>> df = pd.DataFrame(
+ ... {"y": [1, 2, 3]},
+ ... index=pd.to_datetime(
+ ... [
+ ... "2000-03-31 00:00:00",
+ ... "2000-05-31 00:00:00",
+ ... "2000-08-31 00:00:00",
+ ... ]
+ ... ),
+ ... )
>>> df.index.to_period("M")
PeriodIndex(['2000-03', '2000-05', '2000-08'],
dtype='period[M]')
@@ -1232,8 +1240,10 @@ def to_period(self, freq=None) -> PeriodArray:
if freq is None:
freq = self.freqstr or self.inferred_freq
- if isinstance(self.freq, BaseOffset):
- freq = freq_to_period_freqstr(self.freq.n, self.freq.name)
+ if isinstance(self.freq, BaseOffset) and hasattr(
+ self.freq, "_period_dtype_code"
+ ):
+ freq = PeriodDtype(self.freq)._freqstr
if freq is None:
raise ValueError(
@@ -1271,7 +1281,7 @@ def month_name(self, locale=None) -> npt.NDArray[np.object_]:
Examples
--------
- >>> s = pd.Series(pd.date_range(start='2018-01', freq='ME', periods=3))
+ >>> s = pd.Series(pd.date_range(start="2018-01", freq="ME", periods=3))
>>> s
0 2018-01-31
1 2018-02-28
@@ -1283,7 +1293,7 @@ def month_name(self, locale=None) -> npt.NDArray[np.object_]:
2 March
dtype: object
- >>> idx = pd.date_range(start='2018-01', freq='ME', periods=3)
+ >>> idx = pd.date_range(start="2018-01", freq="ME", periods=3)
>>> idx
DatetimeIndex(['2018-01-31', '2018-02-28', '2018-03-31'],
dtype='datetime64[ns]', freq='ME')
@@ -1294,11 +1304,11 @@ def month_name(self, locale=None) -> npt.NDArray[np.object_]:
for example: ``idx.month_name(locale='pt_BR.utf8')`` will return month
names in Brazilian Portuguese language.
- >>> idx = pd.date_range(start='2018-01', freq='ME', periods=3)
+ >>> idx = pd.date_range(start="2018-01", freq="ME", periods=3)
>>> idx
DatetimeIndex(['2018-01-31', '2018-02-28', '2018-03-31'],
dtype='datetime64[ns]', freq='ME')
- >>> idx.month_name(locale='pt_BR.utf8') # doctest: +SKIP
+ >>> idx.month_name(locale="pt_BR.utf8") # doctest: +SKIP
Index(['Janeiro', 'Fevereiro', 'Março'], dtype='object')
"""
values = self._local_timestamps()
@@ -1328,7 +1338,7 @@ def day_name(self, locale=None) -> npt.NDArray[np.object_]:
Examples
--------
- >>> s = pd.Series(pd.date_range(start='2018-01-01', freq='D', periods=3))
+ >>> s = pd.Series(pd.date_range(start="2018-01-01", freq="D", periods=3))
>>> s
0 2018-01-01
1 2018-01-02
@@ -1340,7 +1350,7 @@ def day_name(self, locale=None) -> npt.NDArray[np.object_]:
2 Wednesday
dtype: object
- >>> idx = pd.date_range(start='2018-01-01', freq='D', periods=3)
+ >>> idx = pd.date_range(start="2018-01-01", freq="D", periods=3)
>>> idx
DatetimeIndex(['2018-01-01', '2018-01-02', '2018-01-03'],
dtype='datetime64[ns]', freq='D')
@@ -1351,11 +1361,11 @@ def day_name(self, locale=None) -> npt.NDArray[np.object_]:
for example: ``idx.day_name(locale='pt_BR.utf8')`` will return day
names in Brazilian Portuguese language.
- >>> idx = pd.date_range(start='2018-01-01', freq='D', periods=3)
+ >>> idx = pd.date_range(start="2018-01-01", freq="D", periods=3)
>>> idx
DatetimeIndex(['2018-01-01', '2018-01-02', '2018-01-03'],
dtype='datetime64[ns]', freq='D')
- >>> idx.day_name(locale='pt_BR.utf8') # doctest: +SKIP
+ >>> idx.day_name(locale="pt_BR.utf8") # doctest: +SKIP
Index(['Segunda', 'Terça', 'Quarta'], dtype='object')
"""
values = self._local_timestamps()
@@ -1390,8 +1400,9 @@ def time(self) -> npt.NDArray[np.object_]:
For DatetimeIndex:
- >>> idx = pd.DatetimeIndex(["1/1/2020 10:00:00+00:00",
- ... "2/1/2020 11:00:00+00:00"])
+ >>> idx = pd.DatetimeIndex(
+ ... ["1/1/2020 10:00:00+00:00", "2/1/2020 11:00:00+00:00"]
+ ... )
>>> idx.time
array([datetime.time(10, 0), datetime.time(11, 0)], dtype=object)
"""
@@ -1426,8 +1437,9 @@ def timetz(self) -> npt.NDArray[np.object_]:
For DatetimeIndex:
- >>> idx = pd.DatetimeIndex(["1/1/2020 10:00:00+00:00",
- ... "2/1/2020 11:00:00+00:00"])
+ >>> idx = pd.DatetimeIndex(
+ ... ["1/1/2020 10:00:00+00:00", "2/1/2020 11:00:00+00:00"]
+ ... )
>>> idx.timetz
array([datetime.time(10, 0, tzinfo=datetime.timezone.utc),
datetime.time(11, 0, tzinfo=datetime.timezone.utc)], dtype=object)
@@ -1459,8 +1471,9 @@ def date(self) -> npt.NDArray[np.object_]:
For DatetimeIndex:
- >>> idx = pd.DatetimeIndex(["1/1/2020 10:00:00+00:00",
- ... "2/1/2020 11:00:00+00:00"])
+ >>> idx = pd.DatetimeIndex(
+ ... ["1/1/2020 10:00:00+00:00", "2/1/2020 11:00:00+00:00"]
+ ... )
>>> idx.date
array([datetime.date(2020, 1, 1), datetime.date(2020, 2, 1)], dtype=object)
"""
@@ -1489,7 +1502,7 @@ def isocalendar(self) -> DataFrame:
Examples
--------
- >>> idx = pd.date_range(start='2019-12-29', freq='D', periods=4)
+ >>> idx = pd.date_range(start="2019-12-29", freq="D", periods=4)
>>> idx.isocalendar()
year week day
2019-12-29 2019 52 7
@@ -2125,7 +2138,7 @@ def std(
ddof: int = 1,
keepdims: bool = False,
skipna: bool = True,
- ):
+ ) -> Timedelta:
"""
Return sample standard deviation over requested axis.
@@ -2157,7 +2170,7 @@ def std(
--------
For :class:`pandas.DatetimeIndex`:
- >>> idx = pd.date_range('2001-01-01 00:00', periods=3)
+ >>> idx = pd.date_range("2001-01-01 00:00", periods=3)
>>> idx
DatetimeIndex(['2001-01-01', '2001-01-02', '2001-01-03'],
dtype='datetime64[ns]', freq='D')
@@ -2191,7 +2204,7 @@ def _sequence_to_dt64(
yearfirst: bool = False,
ambiguous: TimeAmbiguous = "raise",
out_unit: str | None = None,
-):
+) -> tuple[np.ndarray, tzinfo | None]:
"""
Parameters
----------
@@ -2360,7 +2373,7 @@ def objects_to_datetime64(
errors: DateTimeErrorChoices = "raise",
allow_object: bool = False,
out_unit: str = "ns",
-):
+) -> tuple[np.ndarray, tzinfo | None]:
"""
Convert data to array of timestamps.
@@ -2371,7 +2384,7 @@ def objects_to_datetime64(
yearfirst : bool
utc : bool, default False
Whether to convert/localize timestamps to UTC.
- errors : {'raise', 'ignore', 'coerce'}
+ errors : {'raise', 'coerce'}
allow_object : bool
Whether to return an object-dtype ndarray instead of raising if the
data contains more than one timezone.
@@ -2391,10 +2404,10 @@ def objects_to_datetime64(
ValueError : if data cannot be converted to datetimes
TypeError : When a type cannot be converted to datetime
"""
- assert errors in ["raise", "ignore", "coerce"]
+ assert errors in ["raise", "coerce"]
# if str-dtype, convert
- data = np.array(data, copy=False, dtype=np.object_)
+ data = np.asarray(data, dtype=np.object_)
result, tz_parsed = tslib.array_to_datetime(
data,
@@ -2665,8 +2678,8 @@ def _infer_tz_from_endpoints(
def _maybe_normalize_endpoints(
- start: Timestamp | None, end: Timestamp | None, normalize: bool
-):
+ start: _TimestampNoneT1, end: _TimestampNoneT2, normalize: bool
+) -> tuple[_TimestampNoneT1, _TimestampNoneT2]:
if normalize:
if start is not None:
start = start.normalize()
@@ -2717,7 +2730,7 @@ def _generate_range(
offset: BaseOffset,
*,
unit: str,
-):
+) -> Generator[Timestamp, None, None]:
"""
Generates a sequence of dates corresponding to the specified time
offset. Similar to dateutil.rrule except uses pandas DateOffset
@@ -2763,12 +2776,12 @@ def _generate_range(
if start and not offset.is_on_offset(start):
# Incompatible types in assignment (expression has type "datetime",
# variable has type "Optional[Timestamp]")
- start = offset.rollforward(start) # type: ignore[assignment]
- elif end and not offset.is_on_offset(end):
- # Incompatible types in assignment (expression has type "datetime",
- # variable has type "Optional[Timestamp]")
- end = offset.rollback(end) # type: ignore[assignment]
+ # GH #56147 account for negative direction and range bounds
+ if offset.n >= 0:
+ start = offset.rollforward(start) # type: ignore[assignment]
+ else:
+ start = offset.rollback(start) # type: ignore[assignment]
# Unsupported operand types for < ("Timestamp" and "None")
if periods is None and end < start and offset.n >= 0: # type: ignore[operator]
diff --git a/pandas/core/arrays/integer.py b/pandas/core/arrays/integer.py
index f9384e25ba9d9..dc453f3e37c50 100644
--- a/pandas/core/arrays/integer.py
+++ b/pandas/core/arrays/integer.py
@@ -115,12 +115,12 @@ class IntegerArray(NumericArray):
String aliases for the dtypes are also available. They are capitalized.
- >>> pd.array([1, None, 3], dtype='Int32')
+ >>> pd.array([1, None, 3], dtype="Int32")
[1, , 3]
Length: 3, dtype: Int32
- >>> pd.array([1, None, 3], dtype='UInt16')
+ >>> pd.array([1, None, 3], dtype="UInt16")
[1, , 3]
Length: 3, dtype: UInt16
diff --git a/pandas/core/arrays/interval.py b/pandas/core/arrays/interval.py
index a19b304529383..af666a591b1bc 100644
--- a/pandas/core/arrays/interval.py
+++ b/pandas/core/arrays/interval.py
@@ -8,11 +8,11 @@
import textwrap
from typing import (
TYPE_CHECKING,
+ Callable,
Literal,
Union,
overload,
)
-import warnings
import numpy as np
@@ -28,7 +28,6 @@
ArrayLike,
AxisInt,
Dtype,
- FillnaOptions,
IntervalClosedType,
NpDtype,
PositionalIndexer,
@@ -79,6 +78,7 @@
unique,
value_counts_internal as value_counts,
)
+from pandas.core.arrays import ArrowExtensionArray
from pandas.core.arrays.base import (
ExtensionArray,
_extension_array_shared_docs,
@@ -121,9 +121,7 @@
}
-_interval_shared_docs[
- "class"
-] = """
+_interval_shared_docs["class"] = """
%(summary)s
Parameters
@@ -232,7 +230,7 @@ def __new__(
dtype: Dtype | None = None,
copy: bool = False,
verify_integrity: bool = True,
- ):
+ ) -> Self:
data = extract_array(data, extract_numpy=True)
if isinstance(data, cls):
@@ -370,11 +368,18 @@ def _ensure_simple_new_inputs(
right = ensure_wrapped_if_datetimelike(right)
right = extract_array(right, extract_numpy=True)
- lbase = getattr(left, "_ndarray", left).base
- rbase = getattr(right, "_ndarray", right).base
- if lbase is not None and lbase is rbase:
- # If these share data, then setitem could corrupt our IA
- right = right.copy()
+ if isinstance(left, ArrowExtensionArray) or isinstance(
+ right, ArrowExtensionArray
+ ):
+ pass
+ else:
+ lbase = getattr(left, "_ndarray", left)
+ lbase = getattr(lbase, "_data", lbase).base
+ rbase = getattr(right, "_ndarray", right)
+ rbase = getattr(rbase, "_data", rbase).base
+ if lbase is not None and lbase is rbase:
+ # If these share data, then setitem could corrupt our IA
+ right = right.copy()
dtype = IntervalDtype(left.dtype, closed=closed)
@@ -699,12 +704,10 @@ def __len__(self) -> int:
return len(self._left)
@overload
- def __getitem__(self, key: ScalarIndexer) -> IntervalOrNA:
- ...
+ def __getitem__(self, key: ScalarIndexer) -> IntervalOrNA: ...
@overload
- def __getitem__(self, key: SequenceIndexer) -> Self:
- ...
+ def __getitem__(self, key: SequenceIndexer) -> Self: ...
def __getitem__(self, key: PositionalIndexer) -> Self | IntervalOrNA:
key = check_array_indexer(self, key)
@@ -889,16 +892,7 @@ def max(self, *, axis: AxisInt | None = None, skipna: bool = True) -> IntervalOr
indexer = obj.argsort()[-1]
return obj[indexer]
- def _pad_or_backfill( # pylint: disable=useless-parent-delegation
- self, *, method: FillnaOptions, limit: int | None = None, copy: bool = True
- ) -> Self:
- # TODO(3.0): after EA.fillna 'method' deprecation is enforced, we can remove
- # this method entirely.
- return super()._pad_or_backfill(method=method, limit=limit, copy=copy)
-
- def fillna(
- self, value=None, method=None, limit: int | None = None, copy: bool = True
- ) -> Self:
+ def fillna(self, value=None, limit: int | None = None, copy: bool = True) -> Self:
"""
Fill NA/NaN values using the specified method.
@@ -909,9 +903,6 @@ def fillna(
Alternatively, a Series or dict can be used to fill in different
values for each index. The value should not be a list. The
value(s) passed should be either Interval objects or NA/NaN.
- method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
- (Not implemented yet for IntervalArray)
- Method to use for filling holes in reindexed Series
limit : int, default None
(Not implemented yet for IntervalArray)
If method is specified, this is the maximum number of consecutive
@@ -932,8 +923,6 @@ def fillna(
"""
if copy is False:
raise NotImplementedError
- if method is not None:
- return super().fillna(value=value, method=method, limit=limit)
value_left, value_right = self._validate_scalar(value)
@@ -1227,21 +1216,14 @@ def value_counts(self, dropna: bool = True) -> Series:
Series.value_counts
"""
# TODO: implement this is a non-naive way!
- with warnings.catch_warnings():
- warnings.filterwarnings(
- "ignore",
- "The behavior of value_counts with object-dtype is deprecated",
- category=FutureWarning,
- )
- result = value_counts(np.asarray(self), dropna=dropna)
- # Once the deprecation is enforced, we will need to do
- # `result.index = result.index.astype(self.dtype)`
+ result = value_counts(np.asarray(self), dropna=dropna)
+ result.index = result.index.astype(self.dtype)
return result
# ---------------------------------------------------------------------
# Rendering Methods
- def _formatter(self, boxed: bool = False):
+ def _formatter(self, boxed: bool = False) -> Callable[[object], str]:
# returning 'str' here causes us to render as e.g. "(0, 1]" instead of
# "Interval(0, 1, closed='right')"
return str
@@ -1481,9 +1463,7 @@ def set_closed(self, closed: IntervalClosedType) -> Self:
dtype = IntervalDtype(left.dtype, closed=closed)
return self._simple_new(left, right, dtype=dtype)
- _interval_shared_docs[
- "is_non_overlapping_monotonic"
- ] = """
+ _interval_shared_docs["is_non_overlapping_monotonic"] = """
Return a boolean whether the %(klass)s is non-overlapping and monotonic.
Non-overlapping means (no Intervals share points), and monotonic means
@@ -1552,7 +1532,9 @@ def is_non_overlapping_monotonic(self) -> bool:
# ---------------------------------------------------------------------
# Conversion
- def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+ def __array__(
+ self, dtype: NpDtype | None = None, copy: bool | None = None
+ ) -> np.ndarray:
"""
Return the IntervalArray's data as a numpy array of Interval
objects (with dtype='object')
@@ -1715,6 +1697,8 @@ def insert(self, loc: int, item: Interval) -> Self:
return self._shallow_copy(new_left, new_right)
def delete(self, loc) -> Self:
+ new_left: np.ndarray | DatetimeArray | TimedeltaArray
+ new_right: np.ndarray | DatetimeArray | TimedeltaArray
if isinstance(self._left, np.ndarray):
new_left = np.delete(self._left, loc)
assert isinstance(self._right, np.ndarray)
@@ -1842,9 +1826,13 @@ def _from_combined(self, combined: np.ndarray) -> IntervalArray:
dtype = self._left.dtype
if needs_i8_conversion(dtype):
assert isinstance(self._left, (DatetimeArray, TimedeltaArray))
- new_left = type(self._left)._from_sequence(nc[:, 0], dtype=dtype)
+ new_left: DatetimeArray | TimedeltaArray | np.ndarray = type(
+ self._left
+ )._from_sequence(nc[:, 0], dtype=dtype)
assert isinstance(self._right, (DatetimeArray, TimedeltaArray))
- new_right = type(self._right)._from_sequence(nc[:, 1], dtype=dtype)
+ new_right: DatetimeArray | TimedeltaArray | np.ndarray = type(
+ self._right
+ )._from_sequence(nc[:, 1], dtype=dtype)
else:
assert isinstance(dtype, np.dtype)
new_left = nc[:, 0].view(dtype)
diff --git a/pandas/core/arrays/masked.py b/pandas/core/arrays/masked.py
index 03c09c5b2fd18..d20d7f98b8aa8 100644
--- a/pandas/core/arrays/masked.py
+++ b/pandas/core/arrays/masked.py
@@ -22,6 +22,7 @@
AxisInt,
DtypeObj,
FillnaOptions,
+ InterpolateOptions,
NpDtype,
PositionalIndexer,
Scalar,
@@ -37,7 +38,6 @@
)
from pandas.errors import AbstractMethodError
from pandas.util._decorators import doc
-from pandas.util._validators import validate_fillna_kwargs
from pandas.core.dtypes.base import ExtensionDtype
from pandas.core.dtypes.common import (
@@ -98,6 +98,8 @@
NumpySorter,
NumpyValueArrayLike,
)
+ from pandas._libs.missing import NAType
+ from pandas.core.arrays import FloatingArray
from pandas.compat.numpy import function as nv
@@ -152,7 +154,7 @@ def _from_sequence(cls, scalars, *, dtype=None, copy: bool = False) -> Self:
@classmethod
@doc(ExtensionArray._empty)
- def _empty(cls, shape: Shape, dtype: ExtensionDtype):
+ def _empty(cls, shape: Shape, dtype: ExtensionDtype) -> Self:
values = np.empty(shape, dtype=dtype.type)
values.fill(cls._internal_fill_value)
mask = np.ones(shape, dtype=bool)
@@ -172,12 +174,10 @@ def dtype(self) -> BaseMaskedDtype:
raise AbstractMethodError(self)
@overload
- def __getitem__(self, item: ScalarIndexer) -> Any:
- ...
+ def __getitem__(self, item: ScalarIndexer) -> Any: ...
@overload
- def __getitem__(self, item: SequenceIndexer) -> Self:
- ...
+ def __getitem__(self, item: SequenceIndexer) -> Self: ...
def __getitem__(self, item: PositionalIndexer) -> Self | Any:
item = check_array_indexer(self, item)
@@ -192,7 +192,12 @@ def __getitem__(self, item: PositionalIndexer) -> Self | Any:
return self._simple_new(self._data[item], newmask)
def _pad_or_backfill(
- self, *, method: FillnaOptions, limit: int | None = None, copy: bool = True
+ self,
+ *,
+ method: FillnaOptions,
+ limit: int | None = None,
+ limit_area: Literal["inside", "outside"] | None = None,
+ copy: bool = True,
) -> Self:
mask = self._mask
@@ -204,7 +209,21 @@ def _pad_or_backfill(
if copy:
npvalues = npvalues.copy()
new_mask = new_mask.copy()
+ elif limit_area is not None:
+ mask = mask.copy()
func(npvalues, limit=limit, mask=new_mask)
+
+ if limit_area is not None and not mask.all():
+ mask = mask.T
+ neg_mask = ~mask
+ first = neg_mask.argmax()
+ last = len(neg_mask) - neg_mask[::-1].argmax() - 1
+ if limit_area == "inside":
+ new_mask[:first] |= mask[:first]
+ new_mask[last + 1 :] |= mask[last + 1 :]
+ elif limit_area == "outside":
+ new_mask[first + 1 : last] |= mask[first + 1 : last]
+
if copy:
return self._simple_new(npvalues.T, new_mask.T)
else:
@@ -217,32 +236,18 @@ def _pad_or_backfill(
return new_values
@doc(ExtensionArray.fillna)
- def fillna(
- self, value=None, method=None, limit: int | None = None, copy: bool = True
- ) -> Self:
- value, method = validate_fillna_kwargs(value, method)
-
+ def fillna(self, value=None, limit: int | None = None, copy: bool = True) -> Self:
mask = self._mask
value = missing.check_value_size(value, mask, len(self))
if mask.any():
- if method is not None:
- func = missing.get_fill_func(method, ndim=self.ndim)
- npvalues = self._data.T
- new_mask = mask.T
- if copy:
- npvalues = npvalues.copy()
- new_mask = new_mask.copy()
- func(npvalues, limit=limit, mask=new_mask)
- return self._simple_new(npvalues.T, new_mask.T)
+ # fill with value
+ if copy:
+ new_values = self.copy()
else:
- # fill with value
- if copy:
- new_values = self.copy()
- else:
- new_values = self[:]
- new_values[mask] = value
+ new_values = self[:]
+ new_values[mask] = value
else:
if copy:
new_values = self.copy()
@@ -281,7 +286,7 @@ def _validate_setitem_value(self, value):
# Note: without the "str" here, the f-string rendering raises in
# py38 builds.
- raise TypeError(f"Invalid value '{str(value)}' for dtype {self.dtype}")
+ raise TypeError(f"Invalid value '{value!s}' for dtype {self.dtype}")
def __setitem__(self, key, value) -> None:
key = check_array_indexer(self, key)
@@ -384,6 +389,8 @@ def round(self, decimals: int = 0, *args, **kwargs):
DataFrame.round : Round values of a DataFrame.
Series.round : Round values of a Series.
"""
+ if self.dtype.kind == "b":
+ return self
nv.validate_round(args, kwargs)
values = np.round(self._data, decimals=decimals, **kwargs)
@@ -407,6 +414,9 @@ def __abs__(self) -> Self:
# ------------------------------------------------------------------
+ def _values_for_json(self) -> np.ndarray:
+ return np.asarray(self, dtype=object)
+
def to_numpy(
self,
dtype: npt.DTypeLike | None = None,
@@ -475,6 +485,8 @@ def to_numpy(
"""
hasna = self._hasna
dtype, na_value = to_numpy_dtype_inference(self, dtype, na_value, hasna)
+ if dtype is None:
+ dtype = object
if hasna:
if (
@@ -499,23 +511,20 @@ def to_numpy(
return data
@doc(ExtensionArray.tolist)
- def tolist(self):
+ def tolist(self) -> list:
if self.ndim > 1:
return [x.tolist() for x in self]
dtype = None if self._hasna else self._data.dtype
return self.to_numpy(dtype=dtype, na_value=libmissing.NA).tolist()
@overload
- def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray:
- ...
+ def astype(self, dtype: npt.DTypeLike, copy: bool = ...) -> np.ndarray: ...
@overload
- def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray:
- ...
+ def astype(self, dtype: ExtensionDtype, copy: bool = ...) -> ExtensionArray: ...
@overload
- def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike:
- ...
+ def astype(self, dtype: AstypeArg, copy: bool = ...) -> ArrayLike: ...
def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
dtype = pandas_dtype(dtype)
@@ -565,7 +574,9 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:
__array_priority__ = 1000 # higher than ndarray so ops dispatch to us
- def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+ def __array__(
+ self, dtype: NpDtype | None = None, copy: bool | None = None
+ ) -> np.ndarray:
"""
the array interface, return my values
We return an object array here to preserve our scalar values
@@ -1069,7 +1080,8 @@ def value_counts(self, dropna: bool = True) -> Series:
arr = IntegerArray(value_counts, mask)
index = Index(
self.dtype.construct_array_type()(
- keys, mask_index # type: ignore[arg-type]
+ keys, # type: ignore[arg-type]
+ mask_index,
)
)
return Series(arr, index=index, name="count", copy=False)
@@ -1305,9 +1317,21 @@ def max(self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs):
return self._wrap_reduction_result("max", result, skipna=skipna, axis=axis)
def map(self, mapper, na_action=None):
- return map_array(self.to_numpy(), mapper, na_action=None)
+ return map_array(self.to_numpy(), mapper, na_action=na_action)
- def any(self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs):
+ @overload
+ def any(
+ self, *, skipna: Literal[True] = ..., axis: AxisInt | None = ..., **kwargs
+ ) -> np.bool_: ...
+
+ @overload
+ def any(
+ self, *, skipna: bool, axis: AxisInt | None = ..., **kwargs
+ ) -> np.bool_ | NAType: ...
+
+ def any(
+ self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs
+ ) -> np.bool_ | NAType:
"""
Return whether any element is truthy.
@@ -1388,7 +1412,19 @@ def any(self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs):
else:
return self.dtype.na_value
- def all(self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs):
+ @overload
+ def all(
+ self, *, skipna: Literal[True] = ..., axis: AxisInt | None = ..., **kwargs
+ ) -> np.bool_: ...
+
+ @overload
+ def all(
+ self, *, skipna: bool, axis: AxisInt | None = ..., **kwargs
+ ) -> np.bool_ | NAType: ...
+
+ def all(
+ self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs
+ ) -> np.bool_ | NAType:
"""
Return whether all elements are truthy.
@@ -1470,6 +1506,58 @@ def all(self, *, skipna: bool = True, axis: AxisInt | None = 0, **kwargs):
else:
return self.dtype.na_value
+ def interpolate(
+ self,
+ *,
+ method: InterpolateOptions,
+ axis: int,
+ index,
+ limit,
+ limit_direction,
+ limit_area,
+ copy: bool,
+ **kwargs,
+ ) -> FloatingArray:
+ """
+ See NDFrame.interpolate.__doc__.
+ """
+ # NB: we return type(self) even if copy=False
+ if self.dtype.kind == "f":
+ if copy:
+ data = self._data.copy()
+ mask = self._mask.copy()
+ else:
+ data = self._data
+ mask = self._mask
+ elif self.dtype.kind in "iu":
+ copy = True
+ data = self._data.astype("f8")
+ mask = self._mask.copy()
+ else:
+ raise NotImplementedError(
+ f"interpolate is not implemented for dtype={self.dtype}"
+ )
+
+ missing.interpolate_2d_inplace(
+ data,
+ method=method,
+ axis=0,
+ index=index,
+ limit=limit,
+ limit_direction=limit_direction,
+ limit_area=limit_area,
+ mask=mask,
+ **kwargs,
+ )
+ if not copy:
+ return self # type: ignore[return-value]
+ if self.dtype.kind == "f":
+ return type(self)._simple_new(data, mask) # type: ignore[return-value]
+ else:
+ from pandas.core.arrays import FloatingArray
+
+ return FloatingArray._simple_new(data, mask)
+
def _accumulate(
self, name: str, *, skipna: bool = True, **kwargs
) -> BaseMaskedArray:
@@ -1541,13 +1629,24 @@ def transpose_homogeneous_masked_arrays(
same dtype. The caller is responsible for ensuring validity of input data.
"""
masked_arrays = list(masked_arrays)
+ dtype = masked_arrays[0].dtype
+
values = [arr._data.reshape(1, -1) for arr in masked_arrays]
- transposed_values = np.concatenate(values, axis=0)
+ transposed_values = np.concatenate(
+ values,
+ axis=0,
+ out=np.empty(
+ (len(masked_arrays), len(masked_arrays[0])),
+ order="F",
+ dtype=dtype.numpy_dtype,
+ ),
+ )
masks = [arr._mask.reshape(1, -1) for arr in masked_arrays]
- transposed_masks = np.concatenate(masks, axis=0)
+ transposed_masks = np.concatenate(
+ masks, axis=0, out=np.empty_like(transposed_values, dtype=bool)
+ )
- dtype = masked_arrays[0].dtype
arr_type = dtype.construct_array_type()
transposed_arrays: list[BaseMaskedArray] = []
for i in range(transposed_values.shape[1]):
diff --git a/pandas/core/arrays/numeric.py b/pandas/core/arrays/numeric.py
index 210450e868698..fe7b32ec9652e 100644
--- a/pandas/core/arrays/numeric.py
+++ b/pandas/core/arrays/numeric.py
@@ -33,12 +33,13 @@
import pyarrow
from pandas._typing import (
- Dtype,
DtypeObj,
Self,
npt,
)
+ from pandas.core.dtypes.dtypes import ExtensionDtype
+
class NumericDtype(BaseMaskedDtype):
_default_np_dtype: np.dtype
@@ -159,7 +160,10 @@ def _coerce_to_data_and_mask(
return values, mask, dtype, inferred_type
original = values
- values = np.array(values, copy=copy)
+ if not copy:
+ values = np.asarray(values)
+ else:
+ values = np.array(values, copy=copy)
inferred_type = None
if values.dtype == object or is_string_dtype(values.dtype):
inferred_type = lib.infer_dtype(values, skipna=True)
@@ -168,7 +172,10 @@ def _coerce_to_data_and_mask(
raise TypeError(f"{values.dtype} cannot be converted to {name}")
elif values.dtype.kind == "b" and checker(dtype):
- values = np.array(values, dtype=default_dtype, copy=copy)
+ if not copy:
+ values = np.asarray(values, dtype=default_dtype)
+ else:
+ values = np.array(values, dtype=default_dtype, copy=copy)
elif values.dtype.kind not in "iuf":
name = dtype_cls.__name__.strip("_")
@@ -207,9 +214,9 @@ def _coerce_to_data_and_mask(
inferred_type not in ["floating", "mixed-integer-float"]
and not mask.any()
):
- values = np.array(original, dtype=dtype, copy=False)
+ values = np.asarray(original, dtype=dtype)
else:
- values = np.array(original, dtype="object", copy=False)
+ values = np.asarray(original, dtype="object")
# we copy as need to coerce here
if mask.any():
@@ -270,7 +277,7 @@ def _coerce_to_array(
@classmethod
def _from_sequence_of_strings(
- cls, strings, *, dtype: Dtype | None = None, copy: bool = False
+ cls, strings, *, dtype: ExtensionDtype, copy: bool = False
) -> Self:
from pandas.core.tools.numeric import to_numeric
diff --git a/pandas/core/arrays/numpy_.py b/pandas/core/arrays/numpy_.py
index d83a37088daec..07eb91e0cb13b 100644
--- a/pandas/core/arrays/numpy_.py
+++ b/pandas/core/arrays/numpy_.py
@@ -150,7 +150,9 @@ def dtype(self) -> NumpyEADtype:
# ------------------------------------------------------------------------
# NumPy Array Interface
- def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+ def __array__(
+ self, dtype: NpDtype | None = None, copy: bool | None = None
+ ) -> np.ndarray:
return np.asarray(self._ndarray, dtype=dtype)
def __array_ufunc__(self, ufunc: np.ufunc, method: str, *inputs, **kwargs):
diff --git a/pandas/core/arrays/period.py b/pandas/core/arrays/period.py
index df5ec356175bf..8baf363b909fb 100644
--- a/pandas/core/arrays/period.py
+++ b/pandas/core/arrays/period.py
@@ -37,7 +37,6 @@
from pandas._libs.tslibs.dtypes import (
FreqGroup,
PeriodDtypeBase,
- freq_to_period_freqstr,
)
from pandas._libs.tslibs.fields import isleapyear_arr
from pandas._libs.tslibs.offsets import (
@@ -55,7 +54,6 @@
cache_readonly,
doc,
)
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.common import (
ensure_object,
@@ -90,6 +88,8 @@
npt,
)
+ from pandas.core.dtypes.dtypes import ExtensionDtype
+
from pandas.core.arrays import (
DatetimeArray,
TimedeltaArray,
@@ -134,11 +134,6 @@ class PeriodArray(dtl.DatelikeOps, libperiod.PeriodMixin): # type: ignore[misc]
dtype : PeriodDtype, optional
A PeriodDtype instance from which to extract a `freq`. If both
`freq` and `dtype` are specified, then the frequencies must match.
- freq : str or DateOffset
- The `freq` to use for the array. Mostly applicable when `values`
- is an ndarray of integers, when `freq` is required. When `values`
- is a PeriodArray (or box around), it's checked that ``values.freq``
- matches `freq`.
copy : bool, default False
Whether to copy the ordinals before storing.
@@ -172,8 +167,7 @@ class PeriodArray(dtl.DatelikeOps, libperiod.PeriodMixin): # type: ignore[misc]
Examples
--------
- >>> pd.arrays.PeriodArray(pd.PeriodIndex(['2023-01-01',
- ... '2023-01-02'], freq='D'))
+ >>> pd.arrays.PeriodArray(pd.PeriodIndex(["2023-01-01", "2023-01-02"], freq="D"))
['2023-01-01', '2023-01-02']
Length: 2, dtype: period[D]
@@ -224,20 +218,7 @@ def _scalar_type(self) -> type[Period]:
# --------------------------------------------------------------------
# Constructors
- def __init__(
- self, values, dtype: Dtype | None = None, freq=None, copy: bool = False
- ) -> None:
- if freq is not None:
- # GH#52462
- warnings.warn(
- "The 'freq' keyword in the PeriodArray constructor is deprecated "
- "and will be removed in a future version. Pass 'dtype' instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- freq = validate_dtype_freq(dtype, freq)
- dtype = PeriodDtype(freq)
-
+ def __init__(self, values, dtype: Dtype | None = None, copy: bool = False) -> None:
if dtype is not None:
dtype = pandas_dtype(dtype)
if not isinstance(dtype, PeriodDtype):
@@ -256,7 +237,10 @@ def __init__(
raise raise_on_incompatible(values, dtype.freq)
values, dtype = values._ndarray, values.dtype
- values = np.array(values, dtype="int64", copy=copy)
+ if not copy:
+ values = np.asarray(values, dtype="int64")
+ else:
+ values = np.array(values, dtype="int64", copy=copy)
if dtype is None:
raise ValueError("dtype is not specified and cannot be inferred")
dtype = cast(PeriodDtype, dtype)
@@ -304,7 +288,7 @@ def _from_sequence(
@classmethod
def _from_sequence_of_strings(
- cls, strings, *, dtype: Dtype | None = None, copy: bool = False
+ cls, strings, *, dtype: ExtensionDtype, copy: bool = False
) -> Self:
return cls._from_sequence(strings, dtype=dtype, copy=copy)
@@ -324,7 +308,7 @@ def _from_datetime64(cls, data, freq, tz=None) -> Self:
PeriodArray[freq]
"""
if isinstance(freq, BaseOffset):
- freq = freq_to_period_freqstr(freq.n, freq.name)
+ freq = PeriodDtype(freq)._freqstr
data, freq = dt64arr_to_periodarr(data, freq, tz)
dtype = PeriodDtype(freq)
return cls(data, dtype=dtype)
@@ -398,9 +382,11 @@ def freq(self) -> BaseOffset:
@property
def freqstr(self) -> str:
- return freq_to_period_freqstr(self.freq.n, self.freq.name)
+ return PeriodDtype(self.freq)._freqstr
- def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+ def __array__(
+ self, dtype: NpDtype | None = None, copy: bool | None = None
+ ) -> np.ndarray:
if dtype == "i8":
return self.asi8
elif dtype == bool:
@@ -637,6 +623,20 @@ def to_timestamp(self, freq=None, how: str = "start") -> DatetimeArray:
>>> idx.to_timestamp()
DatetimeIndex(['2023-01-01', '2023-02-01', '2023-03-01'],
dtype='datetime64[ns]', freq='MS')
+
+ The frequency will not be inferred if the index contains less than
+ three elements, or if the values of index are not strictly monotonic:
+
+ >>> idx = pd.PeriodIndex(["2023-01", "2023-02"], freq="M")
+ >>> idx.to_timestamp()
+ DatetimeIndex(['2023-01-01', '2023-02-01'], dtype='datetime64[ns]', freq=None)
+
+ >>> idx = pd.PeriodIndex(
+ ... ["2023-01", "2023-02", "2023-02", "2023-03"], freq="2M"
+ ... )
+ >>> idx.to_timestamp()
+ DatetimeIndex(['2023-01-01', '2023-02-01', '2023-02-01', '2023-03-01'],
+ dtype='datetime64[ns]', freq=None)
"""
from pandas.core.arrays import DatetimeArray
@@ -719,22 +719,22 @@ def asfreq(self, freq=None, how: str = "E") -> Self:
Examples
--------
- >>> pidx = pd.period_range('2010-01-01', '2015-01-01', freq='Y')
+ >>> pidx = pd.period_range("2010-01-01", "2015-01-01", freq="Y")
>>> pidx
PeriodIndex(['2010', '2011', '2012', '2013', '2014', '2015'],
dtype='period[Y-DEC]')
- >>> pidx.asfreq('M')
+ >>> pidx.asfreq("M")
PeriodIndex(['2010-12', '2011-12', '2012-12', '2013-12', '2014-12',
'2015-12'], dtype='period[M]')
- >>> pidx.asfreq('M', how='S')
+ >>> pidx.asfreq("M", how="S")
PeriodIndex(['2010-01', '2011-01', '2012-01', '2013-01', '2014-01',
'2015-01'], dtype='period[M]')
"""
how = libperiod.validate_end_alias(how)
- if isinstance(freq, BaseOffset):
- freq = freq_to_period_freqstr(freq.n, freq.name)
+ if isinstance(freq, BaseOffset) and hasattr(freq, "_period_dtype_code"):
+ freq = PeriodDtype(freq)._freqstr
freq = Period._maybe_convert_freq(freq)
base1 = self._dtype._dtype_code
@@ -810,30 +810,24 @@ def searchsorted(
return m8arr.searchsorted(npvalue, side=side, sorter=sorter)
def _pad_or_backfill(
- self, *, method: FillnaOptions, limit: int | None = None, copy: bool = True
+ self,
+ *,
+ method: FillnaOptions,
+ limit: int | None = None,
+ limit_area: Literal["inside", "outside"] | None = None,
+ copy: bool = True,
) -> Self:
# view as dt64 so we get treated as timelike in core.missing,
# similar to dtl._period_dispatch
dta = self.view("M8[ns]")
- result = dta._pad_or_backfill(method=method, limit=limit, copy=copy)
+ result = dta._pad_or_backfill(
+ method=method, limit=limit, limit_area=limit_area, copy=copy
+ )
if copy:
return cast("Self", result.view(self.dtype))
else:
return self
- def fillna(
- self, value=None, method=None, limit: int | None = None, copy: bool = True
- ) -> Self:
- if method is not None:
- # view as dt64 so we get treated as timelike in core.missing,
- # similar to dtl._period_dispatch
- dta = self.view("M8[ns]")
- result = dta.fillna(value=value, method=method, limit=limit, copy=copy)
- # error: Incompatible return value type (got "Union[ExtensionArray,
- # ndarray[Any, Any]]", expected "PeriodArray")
- return result.view(self.dtype) # type: ignore[return-value]
- return super().fillna(value=value, method=method, limit=limit, copy=copy)
-
# ------------------------------------------------------------------
# Arithmetic Methods
@@ -980,13 +974,17 @@ def raise_on_incompatible(left, right) -> IncompatibleFrequency:
if isinstance(right, (np.ndarray, ABCTimedeltaArray)) or right is None:
other_freq = None
elif isinstance(right, BaseOffset):
- other_freq = freq_to_period_freqstr(right.n, right.name)
+ with warnings.catch_warnings():
+ warnings.filterwarnings(
+ "ignore", r"PeriodDtype\[B\] is deprecated", category=FutureWarning
+ )
+ other_freq = PeriodDtype(right)._freqstr
elif isinstance(right, (ABCPeriodIndex, PeriodArray, Period)):
other_freq = right.freqstr
else:
other_freq = delta_to_tick(Timedelta(right)).freqstr
- own_freq = freq_to_period_freqstr(left.freq.n, left.freq.name)
+ own_freq = PeriodDtype(left.freq)._freqstr
msg = DIFFERENT_FREQ.format(
cls=type(left).__name__, own_freq=own_freq, other_freq=other_freq
)
@@ -1028,29 +1026,26 @@ def period_array(
Examples
--------
- >>> period_array([pd.Period('2017', freq='Y'),
- ... pd.Period('2018', freq='Y')])
+ >>> period_array([pd.Period("2017", freq="Y"), pd.Period("2018", freq="Y")])
['2017', '2018']
Length: 2, dtype: period[Y-DEC]
- >>> period_array([pd.Period('2017', freq='Y'),
- ... pd.Period('2018', freq='Y'),
- ... pd.NaT])
+ >>> period_array([pd.Period("2017", freq="Y"), pd.Period("2018", freq="Y"), pd.NaT])
['2017', '2018', 'NaT']
Length: 3, dtype: period[Y-DEC]
Integers that look like years are handled
- >>> period_array([2000, 2001, 2002], freq='D')
+ >>> period_array([2000, 2001, 2002], freq="D")
['2000-01-01', '2001-01-01', '2002-01-01']
Length: 3, dtype: period[D]
Datetime-like strings may also be passed
- >>> period_array(['2000-Q1', '2000-Q2', '2000-Q3', '2000-Q4'], freq='Q')
+ >>> period_array(["2000-Q1", "2000-Q2", "2000-Q3", "2000-Q4"], freq="Q")
['2000Q1', '2000Q2', '2000Q3', '2000Q4']
Length: 4, dtype: period[Q-DEC]
@@ -1097,13 +1092,11 @@ def period_array(
@overload
-def validate_dtype_freq(dtype, freq: BaseOffsetT) -> BaseOffsetT:
- ...
+def validate_dtype_freq(dtype, freq: BaseOffsetT) -> BaseOffsetT: ...
@overload
-def validate_dtype_freq(dtype, freq: timedelta | str | None) -> BaseOffset:
- ...
+def validate_dtype_freq(dtype, freq: timedelta | str | None) -> BaseOffset: ...
def validate_dtype_freq(
@@ -1179,12 +1172,7 @@ def dt64arr_to_periodarr(
reso = get_unit_from_dtype(data.dtype)
freq = Period._maybe_convert_freq(freq)
- try:
- base = freq._period_dtype_code
- except (AttributeError, TypeError):
- # AttributeError: _period_dtype_code might not exist
- # TypeError: _period_dtype_code might intentionally raise
- raise TypeError(f"{freq.name} is not supported as period frequency")
+ base = freq._period_dtype_code
return c_dt64arr_to_periodarr(data.view("i8"), base, tz, reso=reso), freq
diff --git a/pandas/core/arrays/sparse/accessor.py b/pandas/core/arrays/sparse/accessor.py
index 3dd7ebf564ca1..58199701647d1 100644
--- a/pandas/core/arrays/sparse/accessor.py
+++ b/pandas/core/arrays/sparse/accessor.py
@@ -1,4 +1,5 @@
"""Sparse accessor"""
+
from __future__ import annotations
from typing import TYPE_CHECKING
@@ -17,6 +18,11 @@
from pandas.core.arrays.sparse.array import SparseArray
if TYPE_CHECKING:
+ from scipy.sparse import (
+ coo_matrix,
+ spmatrix,
+ )
+
from pandas import (
DataFrame,
Series,
@@ -115,7 +121,9 @@ def from_coo(cls, A, dense_index: bool = False) -> Series:
return result
- def to_coo(self, row_levels=(0,), column_levels=(1,), sort_labels: bool = False):
+ def to_coo(
+ self, row_levels=(0,), column_levels=(1,), sort_labels: bool = False
+ ) -> tuple[coo_matrix, list, list]:
"""
Create a scipy.sparse.coo_matrix from a Series with MultiIndex.
@@ -149,7 +157,7 @@ def to_coo(self, row_levels=(0,), column_levels=(1,), sort_labels: bool = False)
... (1, 1, "b", 0),
... (1, 1, "b", 1),
... (2, 1, "b", 0),
- ... (2, 1, "b", 1)
+ ... (2, 1, "b", 1),
... ],
... names=["A", "B", "C", "D"],
... )
@@ -237,8 +245,7 @@ class SparseFrameAccessor(BaseAccessor, PandasDelegate):
Examples
--------
- >>> df = pd.DataFrame({"a": [1, 2, 0, 0],
- ... "b": [3, 0, 0, 4]}, dtype="Sparse[int]")
+ >>> df = pd.DataFrame({"a": [1, 2, 0, 0], "b": [3, 0, 0, 4]}, dtype="Sparse[int]")
>>> df.sparse.density
0.5
"""
@@ -326,7 +333,7 @@ def to_dense(self) -> DataFrame:
data = {k: v.array.to_dense() for k, v in self._parent.items()}
return DataFrame(data, index=self._parent.index, columns=self._parent.columns)
- def to_coo(self):
+ def to_coo(self) -> spmatrix:
"""
Return the contents of the frame as a sparse SciPy COO matrix.
diff --git a/pandas/core/arrays/sparse/array.py b/pandas/core/arrays/sparse/array.py
index 7a3ea85dde2b4..bdcb3219a9875 100644
--- a/pandas/core/arrays/sparse/array.py
+++ b/pandas/core/arrays/sparse/array.py
@@ -1,6 +1,7 @@
"""
SparseArray data structure
"""
+
from __future__ import annotations
from collections import abc
@@ -18,6 +19,8 @@
import numpy as np
+from pandas._config.config import get_option
+
from pandas._libs import lib
import pandas._libs.sparse as splib
from pandas._libs.sparse import (
@@ -95,10 +98,7 @@ class ellipsis(Enum):
from scipy.sparse import spmatrix
- from pandas._typing import (
- FillnaOptions,
- NumpySorter,
- )
+ from pandas._typing import NumpySorter
SparseIndexKind = Literal["integer", "block"]
@@ -454,7 +454,8 @@ def __init__(
# error: Argument "dtype" to "asarray" has incompatible type
# "Union[ExtensionDtype, dtype[Any], None]"; expected "None"
sparse_values = np.asarray(
- data.sp_values, dtype=dtype # type: ignore[arg-type]
+ data.sp_values,
+ dtype=dtype, # type: ignore[arg-type]
)
elif sparse_index is None:
data = extract_array(data, extract_numpy=True)
@@ -551,7 +552,9 @@ def from_spmatrix(cls, data: spmatrix) -> Self:
return cls._simple_new(arr, index, dtype)
- def __array__(self, dtype: NpDtype | None = None) -> np.ndarray:
+ def __array__(
+ self, dtype: NpDtype | None = None, copy: bool | None = None
+ ) -> np.ndarray:
fill_value = self.fill_value
if self.sp_index.ngaps == 0:
@@ -584,11 +587,13 @@ def __setitem__(self, key, value) -> None:
raise TypeError(msg)
@classmethod
- def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
+ def _from_sequence(
+ cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
+ ) -> Self:
return cls(scalars, dtype=dtype)
@classmethod
- def _from_factorized(cls, values, original):
+ def _from_factorized(cls, values, original) -> Self:
return cls(values, dtype=original.dtype)
# ------------------------------------------------------------------------
@@ -665,12 +670,6 @@ def __len__(self) -> int:
def _null_fill_value(self) -> bool:
return self._dtype._is_na_fill_value
- def _fill_value_matches(self, fill_value) -> bool:
- if self._null_fill_value:
- return isna(fill_value)
- else:
- return self.fill_value == fill_value
-
@property
def nbytes(self) -> int:
return self.sp_values.nbytes + self.sp_index.nbytes
@@ -715,17 +714,9 @@ def isna(self) -> Self: # type: ignore[override]
mask[self.sp_index.indices] = isna(self.sp_values)
return type(self)(mask, fill_value=False, dtype=dtype)
- def _pad_or_backfill( # pylint: disable=useless-parent-delegation
- self, *, method: FillnaOptions, limit: int | None = None, copy: bool = True
- ) -> Self:
- # TODO(3.0): We can remove this method once deprecation for fillna method
- # keyword is enforced.
- return super()._pad_or_backfill(method=method, limit=limit, copy=copy)
-
def fillna(
self,
value=None,
- method: FillnaOptions | None = None,
limit: int | None = None,
copy: bool = True,
) -> Self:
@@ -734,17 +725,8 @@ def fillna(
Parameters
----------
- value : scalar, optional
- method : str, optional
-
- .. warning::
-
- Using 'method' will result in high memory use,
- as all `fill_value` methods will be converted to
- an in-memory ndarray
-
+ value : scalar
limit : int, optional
-
copy: bool, default True
Ignored for SparseArray.
@@ -764,22 +746,15 @@ def fillna(
When ``self.fill_value`` is not NA, the result dtype will be
``self.dtype``. Again, this preserves the amount of memory used.
"""
- if (method is None and value is None) or (
- method is not None and value is not None
- ):
- raise ValueError("Must specify one of 'method' or 'value'.")
-
- if method is not None:
- return super().fillna(method=method, limit=limit)
+ if value is None:
+ raise ValueError("Must specify 'value'.")
+ new_values = np.where(isna(self.sp_values), value, self.sp_values)
+ if self._null_fill_value:
+ # This is essentially just updating the dtype.
+ new_dtype = SparseDtype(self.dtype.subtype, fill_value=value)
else:
- new_values = np.where(isna(self.sp_values), value, self.sp_values)
-
- if self._null_fill_value:
- # This is essentially just updating the dtype.
- new_dtype = SparseDtype(self.dtype.subtype, fill_value=value)
- else:
- new_dtype = self.dtype
+ new_dtype = self.dtype
return self._simple_new(new_values, self._sparse_index, new_dtype)
@@ -916,15 +891,13 @@ def value_counts(self, dropna: bool = True) -> Series:
# Indexing
# --------
@overload
- def __getitem__(self, key: ScalarIndexer) -> Any:
- ...
+ def __getitem__(self, key: ScalarIndexer) -> Any: ...
@overload
def __getitem__(
self,
key: SequenceIndexer | tuple[int | ellipsis, ...],
- ) -> Self:
- ...
+ ) -> Self: ...
def __getitem__(
self,
@@ -1144,8 +1117,9 @@ def searchsorted(
side: Literal["left", "right"] = "left",
sorter: NumpySorter | None = None,
) -> npt.NDArray[np.intp] | np.intp:
- msg = "searchsorted requires high memory usage."
- warnings.warn(msg, PerformanceWarning, stacklevel=find_stack_level())
+ if get_option("performance_warnings"):
+ msg = "searchsorted requires high memory usage."
+ warnings.warn(msg, PerformanceWarning, stacklevel=find_stack_level())
v = np.asarray(v)
return np.asarray(self, dtype=self.dtype.subtype).searchsorted(v, side, sorter)
@@ -1241,7 +1215,7 @@ def astype(self, dtype: AstypeArg | None = None, copy: bool = True):
IntIndex
Indices: array([2, 3], dtype=int32)
- >>> arr.astype(SparseDtype(np.dtype('int32')))
+ >>> arr.astype(SparseDtype(np.dtype("int32")))
[0, 0, 1, 2]
Fill: 0
IntIndex
@@ -1250,7 +1224,7 @@ def astype(self, dtype: AstypeArg | None = None, copy: bool = True):
Using a NumPy dtype with a different kind (e.g. float) will coerce
just ``self.sp_values``.
- >>> arr.astype(SparseDtype(np.dtype('float64')))
+ >>> arr.astype(SparseDtype(np.dtype("float64")))
... # doctest: +NORMALIZE_WHITESPACE
[nan, nan, 1.0, 2.0]
Fill: nan
@@ -1901,13 +1875,11 @@ def _make_sparse(
@overload
-def make_sparse_index(length: int, indices, kind: Literal["block"]) -> BlockIndex:
- ...
+def make_sparse_index(length: int, indices, kind: Literal["block"]) -> BlockIndex: ...
@overload
-def make_sparse_index(length: int, indices, kind: Literal["integer"]) -> IntIndex:
- ...
+def make_sparse_index(length: int, indices, kind: Literal["integer"]) -> IntIndex: ...
def make_sparse_index(length: int, indices, kind: SparseIndexKind) -> SparseIndex:
diff --git a/pandas/core/arrays/sparse/scipy_sparse.py b/pandas/core/arrays/sparse/scipy_sparse.py
index 31e09c923d933..cc9fd2d5fb8b0 100644
--- a/pandas/core/arrays/sparse/scipy_sparse.py
+++ b/pandas/core/arrays/sparse/scipy_sparse.py
@@ -3,6 +3,7 @@
Currently only includes to_coo helpers.
"""
+
from __future__ import annotations
from typing import TYPE_CHECKING
diff --git a/pandas/core/arrays/string_.py b/pandas/core/arrays/string_.py
index f451ebc352733..291cc2e62be62 100644
--- a/pandas/core/arrays/string_.py
+++ b/pandas/core/arrays/string_.py
@@ -4,6 +4,7 @@
TYPE_CHECKING,
ClassVar,
Literal,
+ cast,
)
import numpy as np
@@ -91,6 +92,10 @@ class StringDtype(StorageExtensionDtype):
-------
None
+ See Also
+ --------
+ BooleanDtype : Extension dtype for boolean data.
+
Examples
--------
>>> pd.StringDtype()
@@ -257,7 +262,7 @@ class BaseStringArray(ExtensionArray):
"""
@doc(ExtensionArray.tolist)
- def tolist(self):
+ def tolist(self) -> list:
if self.ndim > 1:
return [x.tolist() for x in self]
return list(self.to_numpy())
@@ -314,7 +319,7 @@ class StringArray(BaseStringArray, NumpyExtensionArray): # type: ignore[misc]
See Also
--------
- :func:`pandas.array`
+ :func:`array`
The recommended function for creating a StringArray.
Series.str
The string methods are available on Series backed by
@@ -326,7 +331,7 @@ class StringArray(BaseStringArray, NumpyExtensionArray): # type: ignore[misc]
Examples
--------
- >>> pd.array(['This is', 'some text', None, 'data.'], dtype="string")
+ >>> pd.array(["This is", "some text", None, "data."], dtype="string")
['This is', 'some text', , 'data.']
Length: 4, dtype: string
@@ -334,11 +339,11 @@ class StringArray(BaseStringArray, NumpyExtensionArray): # type: ignore[misc]
Unlike arrays instantiated with ``dtype="object"``, ``StringArray``
will convert the values to strings.
- >>> pd.array(['1', 1], dtype="object")
+ >>> pd.array(["1", 1], dtype="object")
['1', 1]
Length: 2, dtype: object
- >>> pd.array(['1', 1], dtype="string")
+ >>> pd.array(["1", 1], dtype="string")
['1', '1']
Length: 2, dtype: string
@@ -381,7 +386,9 @@ def _validate(self) -> None:
lib.convert_nans_to_NA(self._ndarray)
@classmethod
- def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
+ def _from_sequence(
+ cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
+ ) -> Self:
if dtype and not (isinstance(dtype, str) and dtype == "string"):
dtype = pandas_dtype(dtype)
assert isinstance(dtype, StringDtype) and dtype.storage == "python"
@@ -413,8 +420,8 @@ def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = Fal
@classmethod
def _from_sequence_of_strings(
- cls, strings, *, dtype: Dtype | None = None, copy: bool = False
- ):
+ cls, strings, *, dtype: ExtensionDtype, copy: bool = False
+ ) -> Self:
return cls._from_sequence(strings, dtype=dtype, copy=copy)
@classmethod
@@ -436,7 +443,7 @@ def __arrow_array__(self, type=None):
values[self.isna()] = None
return pa.array(values, type=type, from_pandas=True)
- def _values_for_factorize(self):
+ def _values_for_factorize(self) -> tuple[np.ndarray, None]:
arr = self._ndarray.copy()
mask = self.isna()
arr[mask] = None
@@ -496,10 +503,10 @@ def astype(self, dtype, copy: bool = True):
values = arr.astype(dtype.numpy_dtype)
return IntegerArray(values, mask, copy=False)
elif isinstance(dtype, FloatingDtype):
- arr = self.copy()
+ arr_ea = self.copy()
mask = self.isna()
- arr[mask] = "0"
- values = arr.astype(dtype.numpy_dtype)
+ arr_ea[mask] = "0"
+ values = arr_ea.astype(dtype.numpy_dtype)
return FloatingArray(values, mask, copy=False)
elif isinstance(dtype, ExtensionDtype):
# Skip the NumpyExtensionArray.astype method
@@ -539,7 +546,7 @@ def max(self, axis=None, skipna: bool = True, **kwargs) -> Scalar:
def value_counts(self, dropna: bool = True) -> Series:
from pandas.core.algorithms import value_counts_internal as value_counts
- result = value_counts(self._ndarray, dropna=dropna).astype("Int64")
+ result = value_counts(self._ndarray, sort=False, dropna=dropna).astype("Int64")
result.index = result.index.astype(self.dtype)
return result
@@ -635,7 +642,7 @@ def _str_map(
# error: Argument 1 to "dtype" has incompatible type
# "Union[ExtensionDtype, str, dtype[Any], Type[object]]"; expected
# "Type[object]"
- dtype=np.dtype(dtype), # type: ignore[arg-type]
+ dtype=np.dtype(cast(type, dtype)),
)
if not na_value_is_na:
diff --git a/pandas/core/arrays/string_arrow.py b/pandas/core/arrays/string_arrow.py
index d5a76811a12e6..ec2534ce174ac 100644
--- a/pandas/core/arrays/string_arrow.py
+++ b/pandas/core/arrays/string_arrow.py
@@ -7,11 +7,13 @@
TYPE_CHECKING,
Callable,
Union,
+ cast,
)
-import warnings
import numpy as np
+from pandas._config.config import get_option
+
from pandas._libs import (
lib,
missing as libmissing,
@@ -20,7 +22,6 @@
pa_version_under10p1,
pa_version_under13p0,
)
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.common import (
is_bool_dtype,
@@ -59,9 +60,12 @@
AxisInt,
Dtype,
Scalar,
+ Self,
npt,
)
+ from pandas.core.dtypes.dtypes import ExtensionDtype
+
from pandas import Series
@@ -103,7 +107,7 @@ class ArrowStringArray(ObjectStringArrayMixin, ArrowExtensionArray, BaseStringAr
See Also
--------
- :func:`pandas.array`
+ :func:`array`
The recommended function for creating a ArrowStringArray.
Series.str
The string methods are available on Series backed by
@@ -115,7 +119,7 @@ class ArrowStringArray(ObjectStringArrayMixin, ArrowExtensionArray, BaseStringAr
Examples
--------
- >>> pd.array(['This is', 'some text', None, 'data.'], dtype="string[pyarrow]")
+ >>> pd.array(["This is", "some text", None, "data."], dtype="string[pyarrow]")
['This is', 'some text', , 'data.']
Length: 4, dtype: string
@@ -172,7 +176,9 @@ def __len__(self) -> int:
return len(self._pa_array)
@classmethod
- def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = False):
+ def _from_sequence(
+ cls, scalars, *, dtype: Dtype | None = None, copy: bool = False
+ ) -> Self:
from pandas.core.arrays.masked import BaseMaskedArray
_chk_pyarrow_available()
@@ -200,8 +206,8 @@ def _from_sequence(cls, scalars, *, dtype: Dtype | None = None, copy: bool = Fal
@classmethod
def _from_sequence_of_strings(
- cls, strings, dtype: Dtype | None = None, copy: bool = False
- ):
+ cls, strings, *, dtype: ExtensionDtype, copy: bool = False
+ ) -> Self:
return cls._from_sequence(strings, dtype=dtype, copy=copy)
@property
@@ -268,17 +274,6 @@ def astype(self, dtype, copy: bool = True):
return super().astype(dtype, copy=copy)
- @property
- def _data(self):
- # dask accesses ._data directlys
- warnings.warn(
- f"{type(self).__name__}._data is a deprecated and will be removed "
- "in a future version, use ._pa_array instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self._pa_array
-
# ------------------------------------------------------------------------
# String methods interface
@@ -324,7 +319,7 @@ def _str_map(
# error: Argument 1 to "dtype" has incompatible type
# "Union[ExtensionDtype, str, dtype[Any], Type[object]]"; expected
# "Type[object]"
- dtype=np.dtype(dtype), # type: ignore[arg-type]
+ dtype=np.dtype(cast(type, dtype)),
)
if not na_value_is_na:
@@ -350,7 +345,8 @@ def _str_contains(
self, pat, case: bool = True, flags: int = 0, na=np.nan, regex: bool = True
):
if flags:
- fallback_performancewarning()
+ if get_option("mode.performance_warnings"):
+ fallback_performancewarning()
return super()._str_contains(pat, case, flags, na, regex)
if regex:
@@ -410,7 +406,8 @@ def _str_replace(
regex: bool = True,
):
if isinstance(pat, re.Pattern) or callable(repl) or not case or flags:
- fallback_performancewarning()
+ if get_option("mode.performance_warnings"):
+ fallback_performancewarning()
return super()._str_replace(pat, repl, n, case, flags, regex)
func = pc.replace_substring_regex if regex else pc.replace_substring
@@ -433,13 +430,13 @@ def _str_match(
def _str_fullmatch(
self, pat, case: bool = True, flags: int = 0, na: Scalar | None = None
):
- if not pat.endswith("$") or pat.endswith("//$"):
+ if not pat.endswith("$") or pat.endswith("\\$"):
pat = f"{pat}$"
return self._str_match(pat, case, flags, na)
def _str_slice(
self, start: int | None = None, stop: int | None = None, step: int | None = None
- ):
+ ) -> Self:
if stop is None:
return super()._str_slice(start, stop, step)
if start is None:
@@ -490,27 +487,27 @@ def _str_len(self):
result = pc.utf8_length(self._pa_array)
return self._convert_int_dtype(result)
- def _str_lower(self):
+ def _str_lower(self) -> Self:
return type(self)(pc.utf8_lower(self._pa_array))
- def _str_upper(self):
+ def _str_upper(self) -> Self:
return type(self)(pc.utf8_upper(self._pa_array))
- def _str_strip(self, to_strip=None):
+ def _str_strip(self, to_strip=None) -> Self:
if to_strip is None:
result = pc.utf8_trim_whitespace(self._pa_array)
else:
result = pc.utf8_trim(self._pa_array, characters=to_strip)
return type(self)(result)
- def _str_lstrip(self, to_strip=None):
+ def _str_lstrip(self, to_strip=None) -> Self:
if to_strip is None:
result = pc.utf8_ltrim_whitespace(self._pa_array)
else:
result = pc.utf8_ltrim(self._pa_array, characters=to_strip)
return type(self)(result)
- def _str_rstrip(self, to_strip=None):
+ def _str_rstrip(self, to_strip=None) -> Self:
if to_strip is None:
result = pc.utf8_rtrim_whitespace(self._pa_array)
else:
@@ -637,7 +634,7 @@ def _str_map(
mask.view("uint8"),
convert=False,
na_value=na_value,
- dtype=np.dtype(dtype), # type: ignore[arg-type]
+ dtype=np.dtype(cast(type, dtype)),
)
return result
diff --git a/pandas/core/arrays/timedeltas.py b/pandas/core/arrays/timedeltas.py
index 58455f8cb8398..6eb4d234b349d 100644
--- a/pandas/core/arrays/timedeltas.py
+++ b/pandas/core/arrays/timedeltas.py
@@ -113,7 +113,7 @@ class TimedeltaArray(dtl.TimelikeOps):
Parameters
----------
- values : array-like
+ data : array-like
The timedelta data.
dtype : numpy.dtype
@@ -132,7 +132,7 @@ class TimedeltaArray(dtl.TimelikeOps):
Examples
--------
- >>> pd.arrays.TimedeltaArray._from_sequence(pd.TimedeltaIndex(['1h', '2h']))
+ >>> pd.arrays.TimedeltaArray._from_sequence(pd.TimedeltaIndex(["1h", "2h"]))
['0 days 01:00:00', '0 days 02:00:00']
Length: 2, dtype: timedelta64[ns]
@@ -196,7 +196,6 @@ def dtype(self) -> np.dtype[np.timedelta64]: # type: ignore[override]
# Constructors
_freq = None
- _default_dtype = TD64NS_DTYPE # used in TimeLikeOps.__init__
@classmethod
def _validate_dtype(cls, values, dtype):
@@ -747,7 +746,7 @@ def total_seconds(self) -> npt.NDArray[np.float64]:
--------
**Series**
- >>> s = pd.Series(pd.to_timedelta(np.arange(5), unit='d'))
+ >>> s = pd.Series(pd.to_timedelta(np.arange(5), unit="d"))
>>> s
0 0 days
1 1 days
@@ -766,7 +765,7 @@ def total_seconds(self) -> npt.NDArray[np.float64]:
**TimedeltaIndex**
- >>> idx = pd.to_timedelta(np.arange(5), unit='d')
+ >>> idx = pd.to_timedelta(np.arange(5), unit="d")
>>> idx
TimedeltaIndex(['0 days', '1 days', '2 days', '3 days', '4 days'],
dtype='timedelta64[ns]', freq=None)
@@ -787,7 +786,7 @@ def to_pytimedelta(self) -> npt.NDArray[np.object_]:
Examples
--------
- >>> tdelta_idx = pd.to_timedelta([1, 2, 3], unit='D')
+ >>> tdelta_idx = pd.to_timedelta([1, 2, 3], unit="D")
>>> tdelta_idx
TimedeltaIndex(['1 days', '2 days', '3 days'],
dtype='timedelta64[ns]', freq=None)
@@ -945,7 +944,7 @@ def components(self) -> DataFrame:
Examples
--------
- >>> tdelta_idx = pd.to_timedelta(['1 day 3 min 2 us 42 ns'])
+ >>> tdelta_idx = pd.to_timedelta(["1 day 3 min 2 us 42 ns"])
>>> tdelta_idx
TimedeltaIndex(['1 days 00:03:00.000002042'],
dtype='timedelta64[ns]', freq=None)
@@ -1072,7 +1071,10 @@ def sequence_to_td64ns(
# This includes datetime64-dtype, see GH#23539, GH#29794
raise TypeError(f"dtype {data.dtype} cannot be converted to timedelta64[ns]")
- data = np.array(data, copy=copy)
+ if not copy:
+ data = np.asarray(data)
+ else:
+ data = np.array(data, copy=copy)
assert data.dtype.kind == "m"
assert data.dtype != "m8" # i.e. not unit-less
@@ -1152,7 +1154,7 @@ def _objects_to_td64ns(
higher level.
"""
# coerce Index to np.ndarray, converting string-dtype if necessary
- values = np.array(data, dtype=np.object_, copy=False)
+ values = np.asarray(data, dtype=np.object_)
result = array_to_timedelta64(values, unit=unit, errors=errors)
return result.view("timedelta64[ns]")
diff --git a/pandas/core/base.py b/pandas/core/base.py
index e98f1157572bb..263265701691b 100644
--- a/pandas/core/base.py
+++ b/pandas/core/base.py
@@ -14,12 +14,9 @@
final,
overload,
)
-import warnings
import numpy as np
-from pandas._config import using_copy_on_write
-
from pandas._libs import lib
from pandas._typing import (
AxisInt,
@@ -37,7 +34,6 @@
cache_readonly,
doc,
)
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.cast import can_hold_element
from pandas.core.dtypes.common import (
@@ -89,12 +85,6 @@
_shared_docs: dict[str, str] = {}
-_indexops_doc_kwargs = {
- "klass": "IndexOpsMixin",
- "inplace": "",
- "unique": "IndexOpsMixin",
- "duplicated": "IndexOpsMixin",
-}
class PandasObject(DirNamesMixin):
@@ -106,7 +96,7 @@ class PandasObject(DirNamesMixin):
_cache: dict[str, Any]
@property
- def _constructor(self):
+ def _constructor(self) -> type[Self]:
"""
Class constructor (for this class it's just `__class__`).
"""
@@ -218,7 +208,7 @@ def _obj_with_exclusions(self):
return self.obj
if self._selection is not None:
- return self.obj._getitem_nocopy(self._selection_list)
+ return self.obj[self._selection_list]
if len(self.exclusions) > 0:
# equivalent to `self.obj.drop(self.exclusions, axis=1)
@@ -367,7 +357,7 @@ def ndim(self) -> Literal[1]:
Examples
--------
- >>> s = pd.Series(['Ant', 'Bear', 'Cow'])
+ >>> s = pd.Series(["Ant", "Bear", "Cow"])
>>> s
0 Ant
1 Bear
@@ -409,7 +399,7 @@ def item(self):
For an index:
- >>> s = pd.Series([1], index=['a'])
+ >>> s = pd.Series([1], index=["a"])
>>> s.index.item()
'a'
"""
@@ -426,7 +416,7 @@ def nbytes(self) -> int:
--------
For Series:
- >>> s = pd.Series(['Ant', 'Bear', 'Cow'])
+ >>> s = pd.Series(["Ant", "Bear", "Cow"])
>>> s
0 Ant
1 Bear
@@ -454,7 +444,7 @@ def size(self) -> int:
--------
For Series:
- >>> s = pd.Series(['Ant', 'Bear', 'Cow'])
+ >>> s = pd.Series(["Ant", "Bear", "Cow"])
>>> s
0 Ant
1 Bear
@@ -531,7 +521,7 @@ def array(self) -> ExtensionArray:
For extension types, like Categorical, the actual ExtensionArray
is returned
- >>> ser = pd.Series(pd.Categorical(['a', 'b', 'a']))
+ >>> ser = pd.Series(pd.Categorical(["a", "b", "a"]))
>>> ser.array
['a', 'b', 'a']
Categories (2, object): ['a', 'b']
@@ -568,6 +558,8 @@ def to_numpy(
Returns
-------
numpy.ndarray
+ The NumPy ndarray holding the values from this Series or Index.
+ The dtype of the array may differ. See Notes.
See Also
--------
@@ -610,7 +602,7 @@ def to_numpy(
Examples
--------
- >>> ser = pd.Series(pd.Categorical(['a', 'b', 'a']))
+ >>> ser = pd.Series(pd.Categorical(["a", "b", "a"]))
>>> ser.to_numpy()
array(['a', 'b', 'a'], dtype=object)
@@ -618,7 +610,7 @@ def to_numpy(
Use ``dtype=object`` to return an ndarray of pandas :class:`Timestamp`
objects, each with the correct ``tz``.
- >>> ser = pd.Series(pd.date_range('2000', periods=2, tz="CET"))
+ >>> ser = pd.Series(pd.date_range("2000", periods=2, tz="CET"))
>>> ser.to_numpy(dtype=object)
array([Timestamp('2000-01-01 00:00:00+0100', tz='CET'),
Timestamp('2000-01-02 00:00:00+0100', tz='CET')],
@@ -661,10 +653,10 @@ def to_numpy(
result = np.asarray(values, dtype=dtype)
- if (copy and not fillna) or (not copy and using_copy_on_write()):
+ if (copy and not fillna) or not copy:
if np.shares_memory(self._values[:2], result[:2]):
# Take slices to improve performance of check
- if using_copy_on_write() and not copy:
+ if not copy:
result = result.view()
result.flags.writeable = False
else:
@@ -692,7 +684,8 @@ def argmax(
axis : {{None}}
Unused. Parameter needed for compatibility with DataFrame.
skipna : bool, default True
- Exclude NA/null values when showing the result.
+ Exclude NA/null values. If the entire Series is NA, or if ``skipna=False``
+ and there is an NA value, this method will raise a ``ValueError``.
*args, **kwargs
Additional arguments and keywords for compatibility with NumPy.
@@ -713,8 +706,15 @@ def argmax(
--------
Consider dataset containing cereal calories
- >>> s = pd.Series({{'Corn Flakes': 100.0, 'Almond Delight': 110.0,
- ... 'Cinnamon Toast Crunch': 120.0, 'Cocoa Puff': 110.0}})
+ >>> s = pd.Series(
+ ... [100.0, 110.0, 120.0, 110.0],
+ ... index=[
+ ... "Corn Flakes",
+ ... "Almond Delight",
+ ... "Cinnamon Toast Crunch",
+ ... "Cocoa Puff",
+ ... ],
+ ... )
>>> s
Corn Flakes 100.0
Almond Delight 110.0
@@ -735,28 +735,15 @@ def argmax(
nv.validate_minmax_axis(axis)
skipna = nv.validate_argmax_with_skipna(skipna, args, kwargs)
+ if skipna and len(delegate) > 0 and isna(delegate).all():
+ raise ValueError("Encountered all NA values")
+ elif not skipna and isna(delegate).any():
+ raise ValueError("Encountered an NA value with skipna=False")
+
if isinstance(delegate, ExtensionArray):
- if not skipna and delegate.isna().any():
- warnings.warn(
- f"The behavior of {type(self).__name__}.argmax/argmin "
- "with skipna=False and NAs, or with all-NAs is deprecated. "
- "In a future version this will raise ValueError.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return -1
- else:
- return delegate.argmax()
+ return delegate.argmax()
else:
result = nanops.nanargmax(delegate, skipna=skipna)
- if result == -1:
- warnings.warn(
- f"The behavior of {type(self).__name__}.argmax/argmin "
- "with skipna=False and NAs, or with all-NAs is deprecated. "
- "In a future version this will raise ValueError.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
# error: Incompatible return value type (got "Union[int, ndarray]", expected
# "int")
return result # type: ignore[return-value]
@@ -769,33 +756,20 @@ def argmin(
nv.validate_minmax_axis(axis)
skipna = nv.validate_argmin_with_skipna(skipna, args, kwargs)
+ if skipna and len(delegate) > 0 and isna(delegate).all():
+ raise ValueError("Encountered all NA values")
+ elif not skipna and isna(delegate).any():
+ raise ValueError("Encountered an NA value with skipna=False")
+
if isinstance(delegate, ExtensionArray):
- if not skipna and delegate.isna().any():
- warnings.warn(
- f"The behavior of {type(self).__name__}.argmax/argmin "
- "with skipna=False and NAs, or with all-NAs is deprecated. "
- "In a future version this will raise ValueError.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return -1
- else:
- return delegate.argmin()
+ return delegate.argmin()
else:
result = nanops.nanargmin(delegate, skipna=skipna)
- if result == -1:
- warnings.warn(
- f"The behavior of {type(self).__name__}.argmax/argmin "
- "with skipna=False and NAs, or with all-NAs is deprecated. "
- "In a future version this will raise ValueError.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
# error: Incompatible return value type (got "Union[int, ndarray]", expected
# "int")
return result # type: ignore[return-value]
- def tolist(self):
+ def tolist(self) -> list:
"""
Return a list of the values.
@@ -889,7 +863,7 @@ def hasnans(self) -> bool:
return bool(isna(self).any()) # type: ignore[union-attr]
@final
- def _map_values(self, mapper, na_action=None, convert: bool = True):
+ def _map_values(self, mapper, na_action=None):
"""
An internal function that maps values using the input
correspondence (which can be a dict, Series, or function).
@@ -901,10 +875,6 @@ def _map_values(self, mapper, na_action=None, convert: bool = True):
na_action : {None, 'ignore'}
If 'ignore', propagate NA values, without passing them to the
mapping function
- convert : bool, default True
- Try to find better dtype for elementwise function results. If
- False, leave as dtype=object. Note that the dtype is always
- preserved for some extension array dtypes, such as Categorical.
Returns
-------
@@ -918,7 +888,7 @@ def _map_values(self, mapper, na_action=None, convert: bool = True):
if isinstance(arr, ExtensionArray):
return arr.map(mapper, na_action=na_action)
- return algorithms.map_array(arr, mapper, na_action=na_action, convert=convert)
+ return algorithms.map_array(arr, mapper, na_action=na_action)
@final
def value_counts(
@@ -1068,7 +1038,7 @@ def nunique(self, dropna: bool = True) -> int:
@property
def is_unique(self) -> bool:
"""
- Return boolean if values in the object are unique.
+ Return True if values in the object are unique.
Returns
-------
@@ -1089,7 +1059,7 @@ def is_unique(self) -> bool:
@property
def is_monotonic_increasing(self) -> bool:
"""
- Return boolean if values in the object are monotonically increasing.
+ Return True if values in the object are monotonically increasing.
Returns
-------
@@ -1112,7 +1082,7 @@ def is_monotonic_increasing(self) -> bool:
@property
def is_monotonic_decreasing(self) -> bool:
"""
- Return boolean if values in the object are monotonically decreasing.
+ Return True if values in the object are monotonically decreasing.
Returns
-------
@@ -1164,7 +1134,7 @@ def _memory_usage(self, deep: bool = False) -> int:
24
"""
if hasattr(self.array, "memory_usage"):
- return self.array.memory_usage( # pyright: ignore[reportGeneralTypeIssues]
+ return self.array.memory_usage( # pyright: ignore[reportAttributeAccessIssue]
deep=deep,
)
@@ -1207,9 +1177,7 @@ def factorize(
uniques = Index(uniques)
return codes, uniques
- _shared_docs[
- "searchsorted"
- ] = """
+ _shared_docs["searchsorted"] = """
Find indices where elements should be inserted to maintain order.
Find the indices into a sorted {klass} `self` such that, if the
@@ -1318,8 +1286,7 @@ def searchsorted( # type: ignore[overload-overlap]
value: ScalarLike_co,
side: Literal["left", "right"] = ...,
sorter: NumpySorter = ...,
- ) -> np.intp:
- ...
+ ) -> np.intp: ...
@overload
def searchsorted(
@@ -1327,8 +1294,7 @@ def searchsorted(
value: npt.ArrayLike | ExtensionArray,
side: Literal["left", "right"] = ...,
sorter: NumpySorter = ...,
- ) -> npt.NDArray[np.intp]:
- ...
+ ) -> npt.NDArray[np.intp]: ...
@doc(_shared_docs["searchsorted"], klass="Index")
def searchsorted(
@@ -1356,7 +1322,7 @@ def searchsorted(
sorter=sorter,
)
- def drop_duplicates(self, *, keep: DropKeep = "first"):
+ def drop_duplicates(self, *, keep: DropKeep = "first") -> Self:
duplicated = self._duplicated(keep=keep)
# error: Value of type "IndexOpsMixin" is not indexable
return self[~duplicated] # type: ignore[index]
diff --git a/pandas/core/common.py b/pandas/core/common.py
index 7d864e02be54e..77e986a26fbe9 100644
--- a/pandas/core/common.py
+++ b/pandas/core/common.py
@@ -3,6 +3,7 @@
Note: pandas.core.common is *not* part of the public API.
"""
+
from __future__ import annotations
import builtins
@@ -24,6 +25,7 @@
TYPE_CHECKING,
Any,
Callable,
+ TypeVar,
cast,
overload,
)
@@ -51,7 +53,9 @@
from pandas._typing import (
AnyArrayLike,
ArrayLike,
+ Concatenate,
NpDtype,
+ P,
RandomState,
T,
)
@@ -224,8 +228,7 @@ def asarray_tuplesafe(
@overload
-def asarray_tuplesafe(values: Iterable, dtype: NpDtype | None = ...) -> ArrayLike:
- ...
+def asarray_tuplesafe(values: Iterable, dtype: NpDtype | None = ...) -> ArrayLike: ...
def asarray_tuplesafe(values: Iterable, dtype: NpDtype | None = None) -> ArrayLike:
@@ -233,6 +236,8 @@ def asarray_tuplesafe(values: Iterable, dtype: NpDtype | None = None) -> ArrayLi
values = list(values)
elif isinstance(values, ABCIndex):
return values._values
+ elif isinstance(values, ABCSeries):
+ return values._values
if isinstance(values, list) and dtype in [np.object_, object]:
return construct_1d_object_array_from_listlike(values)
@@ -417,15 +422,13 @@ def standardize_mapping(into):
@overload
-def random_state(state: np.random.Generator) -> np.random.Generator:
- ...
+def random_state(state: np.random.Generator) -> np.random.Generator: ...
@overload
def random_state(
state: int | np.ndarray | np.random.BitGenerator | np.random.RandomState | None,
-) -> np.random.RandomState:
- ...
+) -> np.random.RandomState: ...
def random_state(state: RandomState | None = None):
@@ -463,8 +466,32 @@ def random_state(state: RandomState | None = None):
)
+_T = TypeVar("_T") # Secondary TypeVar for use in pipe's type hints
+
+
+@overload
+def pipe(
+ obj: _T,
+ func: Callable[Concatenate[_T, P], T],
+ *args: P.args,
+ **kwargs: P.kwargs,
+) -> T: ...
+
+
+@overload
def pipe(
- obj, func: Callable[..., T] | tuple[Callable[..., T], str], *args, **kwargs
+ obj: Any,
+ func: tuple[Callable[..., T], str],
+ *args: Any,
+ **kwargs: Any,
+) -> T: ...
+
+
+def pipe(
+ obj: _T,
+ func: Callable[Concatenate[_T, P], T] | tuple[Callable[..., T], str],
+ *args: Any,
+ **kwargs: Any,
) -> T:
"""
Apply a function ``func`` to object ``obj`` either by passing obj as the
@@ -490,12 +517,13 @@ def pipe(
object : the return type of ``func``.
"""
if isinstance(func, tuple):
- func, target = func
+ # Assigning to func_ so pyright understands that it's a callable
+ func_, target = func
if target in kwargs:
msg = f"{target} is both the pipe target and a keyword argument"
raise ValueError(msg)
kwargs[target] = obj
- return func(*args, **kwargs)
+ return func_(*args, **kwargs)
else:
return func(obj, *args, **kwargs)
@@ -576,22 +604,6 @@ def require_length_match(data, index: Index) -> None:
)
-# the ufuncs np.maximum.reduce and np.minimum.reduce default to axis=0,
-# whereas np.min and np.max (which directly call obj.min and obj.max)
-# default to axis=None.
-_builtin_table = {
- builtins.sum: np.sum,
- builtins.max: np.maximum.reduce,
- builtins.min: np.minimum.reduce,
-}
-
-# GH#53425: Only for deprecation
-_builtin_table_alias = {
- builtins.sum: "np.sum",
- builtins.max: "np.maximum.reduce",
- builtins.min: "np.minimum.reduce",
-}
-
_cython_table = {
builtins.sum: "sum",
builtins.max: "max",
@@ -628,14 +640,6 @@ def get_cython_func(arg: Callable) -> str | None:
return _cython_table.get(arg)
-def is_builtin_func(arg):
- """
- if we define a builtin function for this argument, return it,
- otherwise return the arg
- """
- return _builtin_table.get(arg, arg)
-
-
def fill_missing_names(names: Sequence[Hashable | None]) -> list[Hashable]:
"""
If a name is missing then replace it by level_n, where n is the count
diff --git a/pandas/core/computation/align.py b/pandas/core/computation/align.py
index cd852ba9249cf..c5562fb0284b7 100644
--- a/pandas/core/computation/align.py
+++ b/pandas/core/computation/align.py
@@ -1,6 +1,7 @@
"""
Core eval alignment algorithms.
"""
+
from __future__ import annotations
from functools import (
@@ -15,6 +16,8 @@
import numpy as np
+from pandas._config.config import get_option
+
from pandas.errors import PerformanceWarning
from pandas.util._exceptions import find_stack_level
@@ -124,17 +127,21 @@ def _align_core(terms):
reindexer_size = len(reindexer)
ordm = np.log10(max(1, abs(reindexer_size - term_axis_size)))
- if ordm >= 1 and reindexer_size >= 10000:
+ if (
+ get_option("performance_warnings")
+ and ordm >= 1
+ and reindexer_size >= 10000
+ ):
w = (
f"Alignment difference on axis {axis} is larger "
- f"than an order of magnitude on term {repr(terms[i].name)}, "
+ f"than an order of magnitude on term {terms[i].name!r}, "
f"by more than {ordm:.4g}; performance may suffer."
)
warnings.warn(
w, category=PerformanceWarning, stacklevel=find_stack_level()
)
- obj = ti.reindex(reindexer, axis=axis, copy=False)
+ obj = ti.reindex(reindexer, axis=axis)
terms[i].update(obj)
terms[i].update(terms[i].value.values)
diff --git a/pandas/core/computation/engines.py b/pandas/core/computation/engines.py
index a3a05a9d75c6e..5db05ebe33efd 100644
--- a/pandas/core/computation/engines.py
+++ b/pandas/core/computation/engines.py
@@ -1,6 +1,7 @@
"""
Engine classes for :func:`~pandas.eval`
"""
+
from __future__ import annotations
import abc
diff --git a/pandas/core/computation/eval.py b/pandas/core/computation/eval.py
index 6313c2e2c98de..c949cfd1bc657 100644
--- a/pandas/core/computation/eval.py
+++ b/pandas/core/computation/eval.py
@@ -1,10 +1,14 @@
"""
Top level ``eval`` module.
"""
+
from __future__ import annotations
import tokenize
-from typing import TYPE_CHECKING
+from typing import (
+ TYPE_CHECKING,
+ Any,
+)
import warnings
from pandas.util._exceptions import find_stack_level
@@ -177,7 +181,7 @@ def eval(
level: int = 0,
target=None,
inplace: bool = False,
-):
+) -> Any:
"""
Evaluate a Python expression as a string using various backends.
@@ -375,7 +379,7 @@ def eval(
try:
target = env.target
if isinstance(target, NDFrame):
- target = target.copy(deep=None)
+ target = target.copy(deep=False)
else:
target = target.copy()
except AttributeError as err:
@@ -391,7 +395,7 @@ def eval(
if inplace and isinstance(target, NDFrame):
target.loc[:, assigner] = ret
else:
- target[assigner] = ret # pyright: ignore[reportGeneralTypeIssues]
+ target[assigner] = ret # pyright: ignore[reportIndexIssue]
except (TypeError, IndexError) as err:
raise ValueError("Cannot assign expression output to target") from err
diff --git a/pandas/core/computation/expr.py b/pandas/core/computation/expr.py
index b5861fbaebe9c..a8123a898b4fe 100644
--- a/pandas/core/computation/expr.py
+++ b/pandas/core/computation/expr.py
@@ -1,6 +1,7 @@
"""
:func:`~pandas.eval` parsers.
"""
+
from __future__ import annotations
import ast
@@ -695,8 +696,7 @@ def visit_Call(self, node, side=None, **kwargs):
if not isinstance(key, ast.keyword):
# error: "expr" has no attribute "id"
raise ValueError(
- "keyword error in function call "
- f"'{node.func.id}'" # type: ignore[attr-defined]
+ "keyword error in function call " f"'{node.func.id}'" # type: ignore[attr-defined]
)
if key.arg:
diff --git a/pandas/core/computation/expressions.py b/pandas/core/computation/expressions.py
index 6219cac4aeb16..e2acd9a2c97c2 100644
--- a/pandas/core/computation/expressions.py
+++ b/pandas/core/computation/expressions.py
@@ -5,6 +5,7 @@
Offer fast expression evaluation through numexpr
"""
+
from __future__ import annotations
import operator
@@ -214,9 +215,9 @@ def _bool_arith_fallback(op_str, a, b) -> bool:
if _has_bool_dtype(a) and _has_bool_dtype(b):
if op_str in _BOOL_OP_UNSUPPORTED:
warnings.warn(
- f"evaluating in Python space because the {repr(op_str)} "
+ f"evaluating in Python space because the {op_str!r} "
"operator is not supported by numexpr for the bool dtype, "
- f"use {repr(_BOOL_OP_UNSUPPORTED[op_str])} instead.",
+ f"use {_BOOL_OP_UNSUPPORTED[op_str]!r} instead.",
stacklevel=find_stack_level(),
)
return True
diff --git a/pandas/core/computation/ops.py b/pandas/core/computation/ops.py
index 9422434b5cde3..cd9aa1833d586 100644
--- a/pandas/core/computation/ops.py
+++ b/pandas/core/computation/ops.py
@@ -160,7 +160,7 @@ def type(self):
@property
def raw(self) -> str:
- return f"{type(self).__name__}(name={repr(self.name)}, type={self.type})"
+ return f"{type(self).__name__}(name={self.name!r}, type={self.type})"
@property
def is_datetime(self) -> bool:
@@ -320,12 +320,6 @@ def _not_in(x, y):
)
_arith_ops_dict = dict(zip(ARITH_OPS_SYMS, _arith_ops_funcs))
-SPECIAL_CASE_ARITH_OPS_SYMS = ("**", "//", "%")
-_special_case_arith_ops_funcs = (operator.pow, operator.floordiv, operator.mod)
-_special_case_arith_ops_dict = dict(
- zip(SPECIAL_CASE_ARITH_OPS_SYMS, _special_case_arith_ops_funcs)
-)
-
_binary_ops_dict = {}
for d in (_cmp_ops_dict, _bool_ops_dict, _arith_ops_dict):
@@ -387,7 +381,7 @@ def __init__(self, op: str, lhs, rhs) -> None:
# has to be made a list for python3
keys = list(_binary_ops_dict.keys())
raise ValueError(
- f"Invalid binary operator {repr(op)}, valid operators are {keys}"
+ f"Invalid binary operator {op!r}, valid operators are {keys}"
) from err
def __call__(self, env):
@@ -571,7 +565,7 @@ def __init__(self, op: Literal["+", "-", "~", "not"], operand) -> None:
self.func = _unary_ops_dict[op]
except KeyError as err:
raise ValueError(
- f"Invalid unary operator {repr(op)}, "
+ f"Invalid unary operator {op!r}, "
f"valid operators are {UNARY_OPS_SYMS}"
) from err
diff --git a/pandas/core/computation/parsing.py b/pandas/core/computation/parsing.py
index 4cfa0f2baffd5..8fbf8936d31ef 100644
--- a/pandas/core/computation/parsing.py
+++ b/pandas/core/computation/parsing.py
@@ -1,6 +1,7 @@
"""
:func:`~pandas.eval` source string parsing functions
"""
+
from __future__ import annotations
from io import StringIO
diff --git a/pandas/core/computation/pytables.py b/pandas/core/computation/pytables.py
index 04a8ad7ef0be6..39511048abf49 100644
--- a/pandas/core/computation/pytables.py
+++ b/pandas/core/computation/pytables.py
@@ -1,4 +1,5 @@
-""" manage PyTables query interface via Expressions """
+"""manage PyTables query interface via Expressions"""
+
from __future__ import annotations
import ast
@@ -81,7 +82,7 @@ def _resolve_name(self):
if self.side == "left":
# Note: The behavior of __new__ ensures that self.name is a str here
if self.name not in self.env.queryables:
- raise NameError(f"name {repr(self.name)} is not defined")
+ raise NameError(f"name {self.name!r} is not defined")
return self.name
# resolve the rhs (and allow it to be None)
@@ -467,9 +468,7 @@ def visit_Subscript(self, node, **kwargs) -> ops.Term:
try:
return self.const_type(value[slobj], self.env)
except TypeError as err:
- raise ValueError(
- f"cannot subscript {repr(value)} with {repr(slobj)}"
- ) from err
+ raise ValueError(f"cannot subscript {value!r} with {slobj!r}") from err
def visit_Attribute(self, node, **kwargs):
attr = node.attr
diff --git a/pandas/core/computation/scope.py b/pandas/core/computation/scope.py
index 7e553ca448218..7b31e03e58b4b 100644
--- a/pandas/core/computation/scope.py
+++ b/pandas/core/computation/scope.py
@@ -1,6 +1,7 @@
"""
Module for scope operations
"""
+
from __future__ import annotations
from collections import ChainMap
diff --git a/pandas/core/config_init.py b/pandas/core/config_init.py
index 0a9d5af7cbd42..46c9139c3456c 100644
--- a/pandas/core/config_init.py
+++ b/pandas/core/config_init.py
@@ -9,6 +9,7 @@
module is imported, register them here rather than in the module.
"""
+
from __future__ import annotations
import os
@@ -36,7 +37,7 @@
"""
-def use_bottleneck_cb(key) -> None:
+def use_bottleneck_cb(key: str) -> None:
from pandas.core import nanops
nanops.set_use_bottleneck(cf.get_option(key))
@@ -50,7 +51,7 @@ def use_bottleneck_cb(key) -> None:
"""
-def use_numexpr_cb(key) -> None:
+def use_numexpr_cb(key: str) -> None:
from pandas.core.computation import expressions
expressions.set_use_numexpr(cf.get_option(key))
@@ -64,7 +65,7 @@ def use_numexpr_cb(key) -> None:
"""
-def use_numba_cb(key) -> None:
+def use_numba_cb(key: str) -> None:
from pandas.core.util import numba_
numba_.set_use_numba(cf.get_option(key))
@@ -94,11 +95,6 @@ def use_numba_cb(key) -> None:
to ``precision`` in :meth:`numpy.set_printoptions`.
"""
-pc_colspace_doc = """
-: int
- Default space for DataFrame columns.
-"""
-
pc_max_rows_doc = """
: int
If max_rows is exceeded, switch to truncate view. Depending on
@@ -204,11 +200,6 @@ def use_numba_cb(key) -> None:
Enabling this may affect to the performance (default: False)
"""
-pc_ambiguous_as_wide_doc = """
-: boolean
- Whether to handle Unicode characters belong to Ambiguous as Wide (width=2)
- (default: False)
-"""
pc_table_schema_doc = """
: boolean
@@ -286,7 +277,7 @@ def use_numba_cb(key) -> None:
"""
-def table_schema_cb(key) -> None:
+def table_schema_cb(key: str) -> None:
from pandas.io.formats.printing import enable_data_resource_formatter
enable_data_resource_formatter(cf.get_option(key))
@@ -406,61 +397,6 @@ def is_terminal() -> bool:
with cf.config_prefix("mode"):
cf.register_option("sim_interactive", False, tc_sim_interactive_doc)
-use_inf_as_na_doc = """
-: boolean
- True means treat None, NaN, INF, -INF as NA (old way),
- False means None and NaN are null, but INF, -INF are not NA
- (new way).
-
- This option is deprecated in pandas 2.1.0 and will be removed in 3.0.
-"""
-
-# We don't want to start importing everything at the global context level
-# or we'll hit circular deps.
-
-
-def use_inf_as_na_cb(key) -> None:
- # TODO(3.0): enforcing this deprecation will close GH#52501
- from pandas.core.dtypes.missing import _use_inf_as_na
-
- _use_inf_as_na(key)
-
-
-with cf.config_prefix("mode"):
- cf.register_option("use_inf_as_na", False, use_inf_as_na_doc, cb=use_inf_as_na_cb)
-
-cf.deprecate_option(
- # GH#51684
- "mode.use_inf_as_na",
- "use_inf_as_na option is deprecated and will be removed in a future "
- "version. Convert inf values to NaN before operating instead.",
-)
-
-data_manager_doc = """
-: string
- Internal data manager type; can be "block" or "array". Defaults to "block",
- unless overridden by the 'PANDAS_DATA_MANAGER' environment variable (needs
- to be set before pandas is imported).
-"""
-
-
-with cf.config_prefix("mode"):
- cf.register_option(
- "data_manager",
- # Get the default from an environment variable, if set, otherwise defaults
- # to "block". This environment variable can be set for testing.
- os.environ.get("PANDAS_DATA_MANAGER", "block"),
- data_manager_doc,
- validator=is_one_of_factory(["block", "array"]),
- )
-
-cf.deprecate_option(
- # GH#55043
- "mode.data_manager",
- "data_manager option is deprecated and will be removed in a future "
- "version. Only the BlockManager will be available.",
-)
-
# TODO better name?
copy_on_write_doc = """
@@ -499,6 +435,19 @@ def use_inf_as_na_cb(key) -> None:
validator=is_one_of_factory([None, "warn", "raise"]),
)
+performance_warnings = """
+: boolean
+ Whether to show or hide PerformanceWarnings.
+"""
+
+with cf.config_prefix("mode"):
+ cf.register_option(
+ "performance_warnings",
+ True,
+ performance_warnings,
+ validator=is_bool,
+ )
+
string_storage_doc = """
: string
@@ -653,7 +602,7 @@ def use_inf_as_na_cb(key) -> None:
"""
-def register_plotting_backend_cb(key) -> None:
+def register_plotting_backend_cb(key: str | None) -> None:
if key == "matplotlib":
# We defer matplotlib validation, since it's the default
return
@@ -667,7 +616,7 @@ def register_plotting_backend_cb(key) -> None:
"backend",
defval="matplotlib",
doc=plotting_backend_doc,
- validator=register_plotting_backend_cb,
+ validator=register_plotting_backend_cb, # type: ignore[arg-type]
)
@@ -679,7 +628,7 @@ def register_plotting_backend_cb(key) -> None:
"""
-def register_converter_cb(key) -> None:
+def register_converter_cb(key: str) -> None:
from pandas.plotting import (
deregister_matplotlib_converters,
register_matplotlib_converters,
diff --git a/pandas/core/construction.py b/pandas/core/construction.py
index d41a9c80a10ec..ec49340e9a516 100644
--- a/pandas/core/construction.py
+++ b/pandas/core/construction.py
@@ -4,6 +4,7 @@
These should not depend on core.internals.
"""
+
from __future__ import annotations
from collections.abc import Sequence
@@ -14,7 +15,6 @@
cast,
overload,
)
-import warnings
import numpy as np
from numpy import ma
@@ -34,7 +34,6 @@
DtypeObj,
T,
)
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.base import ExtensionDtype
from pandas.core.dtypes.cast import (
@@ -169,7 +168,7 @@ def array(
would no longer return a :class:`arrays.NumpyExtensionArray` backed by a
NumPy array.
- >>> pd.array(['a', 'b'], dtype=str)
+ >>> pd.array(["a", "b"], dtype=str)
['a', 'b']
Length: 2, dtype: str32
@@ -178,7 +177,7 @@ def array(
data. If you really need the new array to be backed by a NumPy array,
specify that in the dtype.
- >>> pd.array(['a', 'b'], dtype=np.dtype(">> pd.array(["a", "b"], dtype=np.dtype("
['a', 'b']
Length: 2, dtype: str32
@@ -193,12 +192,12 @@ def array(
rather than a ``NumpyExtensionArray``. This is for symmetry with the case of
timezone-aware data, which NumPy does not natively support.
- >>> pd.array(['2015', '2016'], dtype='datetime64[ns]')
+ >>> pd.array(["2015", "2016"], dtype="datetime64[ns]")
['2015-01-01 00:00:00', '2016-01-01 00:00:00']
Length: 2, dtype: datetime64[ns]
- >>> pd.array(["1h", "2h"], dtype='timedelta64[ns]')
+ >>> pd.array(["1h", "2h"], dtype="timedelta64[ns]")
['0 days 01:00:00', '0 days 02:00:00']
Length: 2, dtype: timedelta64[ns]
@@ -230,27 +229,27 @@ def array(
>>> with pd.option_context("string_storage", "pyarrow"):
... arr = pd.array(["a", None, "c"])
- ...
>>> arr
['a', , 'c']
Length: 3, dtype: string
- >>> pd.array([pd.Period('2000', freq="D"), pd.Period("2000", freq="D")])
+ >>> pd.array([pd.Period("2000", freq="D"), pd.Period("2000", freq="D")])
['2000-01-01', '2000-01-01']
Length: 2, dtype: period[D]
You can use the string alias for `dtype`
- >>> pd.array(['a', 'b', 'a'], dtype='category')
+ >>> pd.array(["a", "b", "a"], dtype="category")
['a', 'b', 'a']
Categories (2, object): ['a', 'b']
Or specify the actual dtype
- >>> pd.array(['a', 'b', 'a'],
- ... dtype=pd.CategoricalDtype(['a', 'b', 'c'], ordered=True))
+ >>> pd.array(
+ ... ["a", "b", "a"], dtype=pd.CategoricalDtype(["a", "b", "c"], ordered=True)
+ ... )
['a', 'b', 'a']
Categories (3, object): ['a' < 'b' < 'c']
@@ -372,13 +371,10 @@ def array(
return TimedeltaArray._from_sequence(data, dtype=dtype, copy=copy)
elif lib.is_np_dtype(dtype, "mM"):
- warnings.warn(
+ raise ValueError(
+ # GH#53817
r"datetime64 and timedelta64 dtype resolutions other than "
- r"'s', 'ms', 'us', and 'ns' are deprecated. "
- r"In future releases passing unsupported resolutions will "
- r"raise an exception.",
- FutureWarning,
- stacklevel=find_stack_level(),
+ r"'s', 'ms', 'us', and 'ns' are no longer supported."
)
return NumpyExtensionArray._from_sequence(data, dtype=dtype, copy=copy)
@@ -402,15 +398,13 @@ def array(
@overload
def extract_array(
obj: Series | Index, extract_numpy: bool = ..., extract_range: bool = ...
-) -> ArrayLike:
- ...
+) -> ArrayLike: ...
@overload
def extract_array(
obj: T, extract_numpy: bool = ..., extract_range: bool = ...
-) -> T | ArrayLike:
- ...
+) -> T | ArrayLike: ...
def extract_array(
@@ -439,7 +433,7 @@ def extract_array(
Examples
--------
- >>> extract_array(pd.Series(['a', 'b', 'c'], dtype='category'))
+ >>> extract_array(pd.Series(["a", "b", "c"], dtype="category"))
['a', 'b', 'c']
Categories (3, object): ['a', 'b', 'c']
@@ -626,7 +620,10 @@ def sanitize_array(
elif hasattr(data, "__array__"):
# e.g. dask array GH#38645
- data = np.array(data, copy=copy)
+ if not copy:
+ data = np.asarray(data)
+ else:
+ data = np.array(data, copy=copy)
return sanitize_array(
data,
index=index,
@@ -744,8 +741,11 @@ def _sanitize_str_dtypes(
# GH#19853: If data is a scalar, result has already the result
if not lib.is_scalar(data):
if not np.all(isna(data)):
- data = np.array(data, dtype=dtype, copy=False)
- result = np.array(data, dtype=object, copy=copy)
+ data = np.asarray(data, dtype=dtype)
+ if not copy:
+ result = np.asarray(data, dtype=object)
+ else:
+ result = np.array(data, dtype=object, copy=copy)
return result
@@ -810,6 +810,8 @@ def _try_cast(
# this will raise if we have e.g. floats
subarr = maybe_cast_to_integer_array(arr, dtype)
+ elif not copy:
+ subarr = np.asarray(arr, dtype=dtype)
else:
subarr = np.array(arr, dtype=dtype, copy=copy)
diff --git a/pandas/core/dtypes/api.py b/pandas/core/dtypes/api.py
index 254abe330b8e7..e66104d6afcd9 100644
--- a/pandas/core/dtypes/api.py
+++ b/pandas/core/dtypes/api.py
@@ -20,7 +20,6 @@
is_int64_dtype,
is_integer,
is_integer_dtype,
- is_interval,
is_interval_dtype,
is_iterator,
is_list_like,
@@ -63,7 +62,6 @@
"is_int64_dtype",
"is_integer",
"is_integer_dtype",
- "is_interval",
"is_interval_dtype",
"is_iterator",
"is_list_like",
diff --git a/pandas/core/dtypes/astype.py b/pandas/core/dtypes/astype.py
index f5579082c679b..086f7d2da6640 100644
--- a/pandas/core/dtypes/astype.py
+++ b/pandas/core/dtypes/astype.py
@@ -2,6 +2,7 @@
Functions for implementing 'astype' methods according to pandas conventions,
particularly ones that differ from numpy.
"""
+
from __future__ import annotations
import inspect
@@ -36,21 +37,17 @@
from pandas.core.arrays import ExtensionArray
-_dtype_obj = np.dtype(object)
-
@overload
def _astype_nansafe(
arr: np.ndarray, dtype: np.dtype, copy: bool = ..., skipna: bool = ...
-) -> np.ndarray:
- ...
+) -> np.ndarray: ...
@overload
def _astype_nansafe(
arr: np.ndarray, dtype: ExtensionDtype, copy: bool = ..., skipna: bool = ...
-) -> ExtensionArray:
- ...
+) -> ExtensionArray: ...
def _astype_nansafe(
@@ -258,6 +255,11 @@ def astype_is_view(dtype: DtypeObj, new_dtype: DtypeObj) -> bool:
-------
True if new data is a view or not guaranteed to be a copy, False otherwise
"""
+ if dtype.kind in "iufb" and dtype.kind == new_dtype.kind:
+ # fastpath for numeric dtypes
+ if hasattr(dtype, "itemsize") and hasattr(new_dtype, "itemsize"):
+ return dtype.itemsize == new_dtype.itemsize # pyright: ignore[reportAttributeAccessIssue]
+
if isinstance(dtype, np.dtype) and not isinstance(new_dtype, np.dtype):
new_dtype, dtype = dtype, new_dtype
diff --git a/pandas/core/dtypes/base.py b/pandas/core/dtypes/base.py
index 6b00a5284ec5b..2f8e59cd6e89c 100644
--- a/pandas/core/dtypes/base.py
+++ b/pandas/core/dtypes/base.py
@@ -1,6 +1,7 @@
"""
Extend pandas with custom array types.
"""
+
from __future__ import annotations
from typing import (
@@ -96,10 +97,8 @@ class property**.
>>> from pandas.api.extensions import ExtensionArray
>>> class ExtensionDtype:
... def __from_arrow__(
- ... self,
- ... array: pyarrow.Array | pyarrow.ChunkedArray
- ... ) -> ExtensionArray:
- ... ...
+ ... self, array: pyarrow.Array | pyarrow.ChunkedArray
+ ... ) -> ExtensionArray: ...
This class does not inherit from 'abc.ABCMeta' for performance reasons.
Methods and properties required by the interface raise
@@ -529,22 +528,18 @@ def register(self, dtype: type_t[ExtensionDtype]) -> None:
self.dtypes.append(dtype)
@overload
- def find(self, dtype: type_t[ExtensionDtypeT]) -> type_t[ExtensionDtypeT]:
- ...
+ def find(self, dtype: type_t[ExtensionDtypeT]) -> type_t[ExtensionDtypeT]: ...
@overload
- def find(self, dtype: ExtensionDtypeT) -> ExtensionDtypeT:
- ...
+ def find(self, dtype: ExtensionDtypeT) -> ExtensionDtypeT: ...
@overload
- def find(self, dtype: str) -> ExtensionDtype | None:
- ...
+ def find(self, dtype: str) -> ExtensionDtype | None: ...
@overload
def find(
self, dtype: npt.DTypeLike
- ) -> type_t[ExtensionDtype] | ExtensionDtype | None:
- ...
+ ) -> type_t[ExtensionDtype] | ExtensionDtype | None: ...
def find(
self, dtype: type_t[ExtensionDtype] | ExtensionDtype | npt.DTypeLike
diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py
index d7a177c2a19c0..a130983337f64 100644
--- a/pandas/core/dtypes/cast.py
+++ b/pandas/core/dtypes/cast.py
@@ -243,17 +243,19 @@ def _disallow_mismatched_datetimelike(value, dtype: DtypeObj) -> None:
elif (vdtype.kind == "m" and dtype.kind == "M") or (
vdtype.kind == "M" and dtype.kind == "m"
):
- raise TypeError(f"Cannot cast {repr(value)} to {dtype}")
+ raise TypeError(f"Cannot cast {value!r} to {dtype}")
@overload
-def maybe_downcast_to_dtype(result: np.ndarray, dtype: str | np.dtype) -> np.ndarray:
- ...
+def maybe_downcast_to_dtype(
+ result: np.ndarray, dtype: str | np.dtype
+) -> np.ndarray: ...
@overload
-def maybe_downcast_to_dtype(result: ExtensionArray, dtype: str | np.dtype) -> ArrayLike:
- ...
+def maybe_downcast_to_dtype(
+ result: ExtensionArray, dtype: str | np.dtype
+) -> ArrayLike: ...
def maybe_downcast_to_dtype(result: ArrayLike, dtype: str | np.dtype) -> ArrayLike:
@@ -317,15 +319,13 @@ def maybe_downcast_to_dtype(result: ArrayLike, dtype: str | np.dtype) -> ArrayLi
@overload
def maybe_downcast_numeric(
result: np.ndarray, dtype: np.dtype, do_round: bool = False
-) -> np.ndarray:
- ...
+) -> np.ndarray: ...
@overload
def maybe_downcast_numeric(
result: ExtensionArray, dtype: DtypeObj, do_round: bool = False
-) -> ArrayLike:
- ...
+) -> ArrayLike: ...
def maybe_downcast_numeric(
@@ -513,13 +513,11 @@ def _maybe_cast_to_extension_array(
@overload
-def ensure_dtype_can_hold_na(dtype: np.dtype) -> np.dtype:
- ...
+def ensure_dtype_can_hold_na(dtype: np.dtype) -> np.dtype: ...
@overload
-def ensure_dtype_can_hold_na(dtype: ExtensionDtype) -> ExtensionDtype:
- ...
+def ensure_dtype_can_hold_na(dtype: ExtensionDtype) -> ExtensionDtype: ...
def ensure_dtype_can_hold_na(dtype: DtypeObj) -> DtypeObj:
@@ -589,7 +587,9 @@ def maybe_promote(dtype: np.dtype, fill_value=np.nan):
# error: Argument 3 to "__call__" of "_lru_cache_wrapper" has incompatible type
# "Type[Any]"; expected "Hashable" [arg-type]
dtype, fill_value = _maybe_promote_cached(
- dtype, fill_value, type(fill_value) # type: ignore[arg-type]
+ dtype,
+ fill_value,
+ type(fill_value), # type: ignore[arg-type]
)
except TypeError:
# if fill_value is not hashable (required for caching)
@@ -891,10 +891,10 @@ def infer_dtype_from_array(arr) -> tuple[DtypeObj, ArrayLike]:
Examples
--------
- >>> np.asarray([1, '1'])
+ >>> np.asarray([1, "1"])
array(['1', '1'], dtype='>> infer_dtype_from_array([1, '1'])
+ >>> infer_dtype_from_array([1, "1"])
(dtype('O'), [1, '1'])
"""
if isinstance(arr, np.ndarray):
@@ -1416,18 +1416,15 @@ def np_find_common_type(*dtypes: np.dtype) -> np.dtype:
@overload
-def find_common_type(types: list[np.dtype]) -> np.dtype:
- ...
+def find_common_type(types: list[np.dtype]) -> np.dtype: ...
@overload
-def find_common_type(types: list[ExtensionDtype]) -> DtypeObj:
- ...
+def find_common_type(types: list[ExtensionDtype]) -> DtypeObj: ...
@overload
-def find_common_type(types: list[DtypeObj]) -> DtypeObj:
- ...
+def find_common_type(types: list[DtypeObj]) -> DtypeObj: ...
def find_common_type(types):
@@ -1501,7 +1498,10 @@ def construct_2d_arraylike_from_scalar(
# Attempt to coerce to a numpy array
try:
- arr = np.array(value, dtype=dtype, copy=copy)
+ if not copy:
+ arr = np.asarray(value, dtype=dtype)
+ else:
+ arr = np.array(value, dtype=dtype, copy=copy)
except (ValueError, TypeError) as err:
raise TypeError(
f"DataFrame constructor called with incompatible data and dtype: {err}"
@@ -1541,25 +1541,24 @@ def construct_1d_arraylike_from_scalar(
if isinstance(dtype, ExtensionDtype):
cls = dtype.construct_array_type()
seq = [] if length == 0 else [value]
- subarr = cls._from_sequence(seq, dtype=dtype).repeat(length)
+ return cls._from_sequence(seq, dtype=dtype).repeat(length)
+
+ if length and dtype.kind in "iu" and isna(value):
+ # coerce if we have nan for an integer dtype
+ dtype = np.dtype("float64")
+ elif lib.is_np_dtype(dtype, "US"):
+ # we need to coerce to object dtype to avoid
+ # to allow numpy to take our string as a scalar value
+ dtype = np.dtype("object")
+ if not isna(value):
+ value = ensure_str(value)
+ elif dtype.kind in "mM":
+ value = _maybe_box_and_unbox_datetimelike(value, dtype)
- else:
- if length and dtype.kind in "iu" and isna(value):
- # coerce if we have nan for an integer dtype
- dtype = np.dtype("float64")
- elif lib.is_np_dtype(dtype, "US"):
- # we need to coerce to object dtype to avoid
- # to allow numpy to take our string as a scalar value
- dtype = np.dtype("object")
- if not isna(value):
- value = ensure_str(value)
- elif dtype.kind in "mM":
- value = _maybe_box_and_unbox_datetimelike(value, dtype)
-
- subarr = np.empty(length, dtype=dtype)
- if length:
- # GH 47391: numpy > 1.24 will raise filling np.nan into int dtypes
- subarr.fill(value)
+ subarr = np.empty(length, dtype=dtype)
+ if length:
+ # GH 47391: numpy > 1.24 will raise filling np.nan into int dtypes
+ subarr.fill(value)
return subarr
@@ -1651,7 +1650,7 @@ def maybe_cast_to_integer_array(arr: list | np.ndarray, dtype: np.dtype) -> np.n
"out-of-bound Python int",
DeprecationWarning,
)
- casted = np.array(arr, dtype=dtype, copy=False)
+ casted = np.asarray(arr, dtype=dtype)
else:
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=RuntimeWarning)
@@ -1682,6 +1681,7 @@ def maybe_cast_to_integer_array(arr: list | np.ndarray, dtype: np.dtype) -> np.n
arr = np.asarray(arr)
if np.issubdtype(arr.dtype, str):
+ # TODO(numpy-2.0 min): This case will raise an OverflowError above
if (casted.astype(str) == arr).all():
return casted
raise ValueError(f"string values cannot be losslessly cast to {dtype}")
@@ -1816,8 +1816,8 @@ def np_can_hold_element(dtype: np.dtype, element: Any) -> Any:
# TODO: general-case for EAs?
try:
casted = element.astype(dtype)
- except (ValueError, TypeError):
- raise LossySetitemError
+ except (ValueError, TypeError) as err:
+ raise LossySetitemError from err
# Check for cases of either
# a) lossy overflow/rounding or
# b) semantic changes like dt64->int64
diff --git a/pandas/core/dtypes/common.py b/pandas/core/dtypes/common.py
index 2245359fd8eac..aa621fea6c39a 100644
--- a/pandas/core/dtypes/common.py
+++ b/pandas/core/dtypes/common.py
@@ -1,6 +1,7 @@
"""
Common type operations.
"""
+
from __future__ import annotations
from typing import (
@@ -42,7 +43,6 @@
is_float,
is_hashable,
is_integer,
- is_interval,
is_iterator,
is_list_like,
is_named_tuple,
@@ -169,6 +169,9 @@ def is_sparse(arr) -> bool:
"""
Check whether an array-like is a 1-D pandas sparse array.
+ .. deprecated:: 2.1.0
+ Use isinstance(dtype, pd.SparseDtype) instead.
+
Check that the one-dimensional array-like is a pandas sparse array.
Returns True if it is a pandas sparse array, not another type of
sparse array.
@@ -295,6 +298,9 @@ def is_datetime64tz_dtype(arr_or_dtype) -> bool:
"""
Check whether an array-like or dtype is of a DatetimeTZDtype dtype.
+ .. deprecated:: 2.1.0
+ Use isinstance(dtype, pd.DatetimeTZDtype) instead.
+
Parameters
----------
arr_or_dtype : array-like or dtype
@@ -367,7 +373,7 @@ def is_timedelta64_dtype(arr_or_dtype) -> bool:
False
>>> is_timedelta64_dtype(pd.Series([], dtype="timedelta64[ns]"))
True
- >>> is_timedelta64_dtype('0 days')
+ >>> is_timedelta64_dtype("0 days")
False
"""
if isinstance(arr_or_dtype, np.dtype):
@@ -381,6 +387,9 @@ def is_period_dtype(arr_or_dtype) -> bool:
"""
Check whether an array-like or dtype is of the Period dtype.
+ .. deprecated:: 2.2.0
+ Use isinstance(dtype, pd.Period) instead.
+
Parameters
----------
arr_or_dtype : array-like or dtype
@@ -424,6 +433,9 @@ def is_interval_dtype(arr_or_dtype) -> bool:
"""
Check whether an array-like or dtype is of the Interval dtype.
+ .. deprecated:: 2.2.0
+ Use isinstance(dtype, pd.IntervalDtype) instead.
+
Parameters
----------
arr_or_dtype : array-like or dtype
@@ -470,6 +482,9 @@ def is_categorical_dtype(arr_or_dtype) -> bool:
"""
Check whether an array-like or dtype is of the Categorical dtype.
+ .. deprecated:: 2.2.0
+ Use isinstance(dtype, pd.CategoricalDtype) instead.
+
Parameters
----------
arr_or_dtype : array-like or dtype
@@ -544,7 +559,7 @@ def is_string_dtype(arr_or_dtype) -> bool:
True
>>> is_string_dtype(int)
False
- >>> is_string_dtype(np.array(['a', 'b']))
+ >>> is_string_dtype(np.array(["a", "b"]))
True
>>> is_string_dtype(pd.Series([1, 2]))
False
@@ -646,9 +661,9 @@ def is_integer_dtype(arr_or_dtype) -> bool:
False
>>> is_integer_dtype(np.uint64)
True
- >>> is_integer_dtype('int8')
+ >>> is_integer_dtype("int8")
True
- >>> is_integer_dtype('Int8')
+ >>> is_integer_dtype("Int8")
True
>>> is_integer_dtype(pd.Int8Dtype)
True
@@ -656,13 +671,13 @@ def is_integer_dtype(arr_or_dtype) -> bool:
False
>>> is_integer_dtype(np.timedelta64)
False
- >>> is_integer_dtype(np.array(['a', 'b']))
+ >>> is_integer_dtype(np.array(["a", "b"]))
False
>>> is_integer_dtype(pd.Series([1, 2]))
True
>>> is_integer_dtype(np.array([], dtype=np.timedelta64))
False
- >>> is_integer_dtype(pd.Index([1, 2.])) # float
+ >>> is_integer_dtype(pd.Index([1, 2.0])) # float
False
"""
return _is_dtype_type(
@@ -703,9 +718,9 @@ def is_signed_integer_dtype(arr_or_dtype) -> bool:
False
>>> is_signed_integer_dtype(np.uint64) # unsigned
False
- >>> is_signed_integer_dtype('int8')
+ >>> is_signed_integer_dtype("int8")
True
- >>> is_signed_integer_dtype('Int8')
+ >>> is_signed_integer_dtype("Int8")
True
>>> is_signed_integer_dtype(pd.Int8Dtype)
True
@@ -713,13 +728,13 @@ def is_signed_integer_dtype(arr_or_dtype) -> bool:
False
>>> is_signed_integer_dtype(np.timedelta64)
False
- >>> is_signed_integer_dtype(np.array(['a', 'b']))
+ >>> is_signed_integer_dtype(np.array(["a", "b"]))
False
>>> is_signed_integer_dtype(pd.Series([1, 2]))
True
>>> is_signed_integer_dtype(np.array([], dtype=np.timedelta64))
False
- >>> is_signed_integer_dtype(pd.Index([1, 2.])) # float
+ >>> is_signed_integer_dtype(pd.Index([1, 2.0])) # float
False
>>> is_signed_integer_dtype(np.array([1, 2], dtype=np.uint32)) # unsigned
False
@@ -759,17 +774,17 @@ def is_unsigned_integer_dtype(arr_or_dtype) -> bool:
False
>>> is_unsigned_integer_dtype(np.uint64)
True
- >>> is_unsigned_integer_dtype('uint8')
+ >>> is_unsigned_integer_dtype("uint8")
True
- >>> is_unsigned_integer_dtype('UInt8')
+ >>> is_unsigned_integer_dtype("UInt8")
True
>>> is_unsigned_integer_dtype(pd.UInt8Dtype)
True
- >>> is_unsigned_integer_dtype(np.array(['a', 'b']))
+ >>> is_unsigned_integer_dtype(np.array(["a", "b"]))
False
>>> is_unsigned_integer_dtype(pd.Series([1, 2])) # signed
False
- >>> is_unsigned_integer_dtype(pd.Index([1, 2.])) # float
+ >>> is_unsigned_integer_dtype(pd.Index([1, 2.0])) # float
False
>>> is_unsigned_integer_dtype(np.array([1, 2], dtype=np.uint32))
True
@@ -815,9 +830,9 @@ def is_int64_dtype(arr_or_dtype) -> bool:
False
>>> is_int64_dtype(np.int64) # doctest: +SKIP
True
- >>> is_int64_dtype('int8') # doctest: +SKIP
+ >>> is_int64_dtype("int8") # doctest: +SKIP
False
- >>> is_int64_dtype('Int8') # doctest: +SKIP
+ >>> is_int64_dtype("Int8") # doctest: +SKIP
False
>>> is_int64_dtype(pd.Int64Dtype) # doctest: +SKIP
True
@@ -825,11 +840,11 @@ def is_int64_dtype(arr_or_dtype) -> bool:
False
>>> is_int64_dtype(np.uint64) # unsigned # doctest: +SKIP
False
- >>> is_int64_dtype(np.array(['a', 'b'])) # doctest: +SKIP
+ >>> is_int64_dtype(np.array(["a", "b"])) # doctest: +SKIP
False
>>> is_int64_dtype(np.array([1, 2], dtype=np.int64)) # doctest: +SKIP
True
- >>> is_int64_dtype(pd.Index([1, 2.])) # float # doctest: +SKIP
+ >>> is_int64_dtype(pd.Index([1, 2.0])) # float # doctest: +SKIP
False
>>> is_int64_dtype(np.array([1, 2], dtype=np.uint32)) # unsigned # doctest: +SKIP
False
@@ -870,7 +885,7 @@ def is_datetime64_any_dtype(arr_or_dtype) -> bool:
True
>>> is_datetime64_any_dtype(DatetimeTZDtype("ns", "US/Eastern"))
True
- >>> is_datetime64_any_dtype(np.array(['a', 'b']))
+ >>> is_datetime64_any_dtype(np.array(["a", "b"]))
False
>>> is_datetime64_any_dtype(np.array([1, 2]))
False
@@ -890,7 +905,11 @@ def is_datetime64_any_dtype(arr_or_dtype) -> bool:
tipo = _get_dtype(arr_or_dtype)
except TypeError:
return False
- return lib.is_np_dtype(tipo, "M") or isinstance(tipo, DatetimeTZDtype)
+ return (
+ lib.is_np_dtype(tipo, "M")
+ or isinstance(tipo, DatetimeTZDtype)
+ or (isinstance(tipo, ExtensionDtype) and tipo.kind == "M")
+ )
def is_datetime64_ns_dtype(arr_or_dtype) -> bool:
@@ -919,7 +938,7 @@ def is_datetime64_ns_dtype(arr_or_dtype) -> bool:
False
>>> is_datetime64_ns_dtype(DatetimeTZDtype("ns", "US/Eastern"))
True
- >>> is_datetime64_ns_dtype(np.array(['a', 'b']))
+ >>> is_datetime64_ns_dtype(np.array(["a", "b"]))
False
>>> is_datetime64_ns_dtype(np.array([1, 2]))
False
@@ -961,11 +980,11 @@ def is_timedelta64_ns_dtype(arr_or_dtype) -> bool:
Examples
--------
>>> from pandas.core.dtypes.common import is_timedelta64_ns_dtype
- >>> is_timedelta64_ns_dtype(np.dtype('m8[ns]'))
+ >>> is_timedelta64_ns_dtype(np.dtype("m8[ns]"))
True
- >>> is_timedelta64_ns_dtype(np.dtype('m8[ps]')) # Wrong frequency
+ >>> is_timedelta64_ns_dtype(np.dtype("m8[ps]")) # Wrong frequency
False
- >>> is_timedelta64_ns_dtype(np.array([1, 2], dtype='m8[ns]'))
+ >>> is_timedelta64_ns_dtype(np.array([1, 2], dtype="m8[ns]"))
True
>>> is_timedelta64_ns_dtype(np.array([1, 2], dtype=np.timedelta64))
False
@@ -1047,7 +1066,7 @@ def needs_i8_conversion(dtype: DtypeObj | None) -> bool:
False
>>> needs_i8_conversion(np.dtype(np.datetime64))
True
- >>> needs_i8_conversion(np.array(['a', 'b']))
+ >>> needs_i8_conversion(np.array(["a", "b"]))
False
>>> needs_i8_conversion(pd.Series([1, 2]))
False
@@ -1092,11 +1111,11 @@ def is_numeric_dtype(arr_or_dtype) -> bool:
False
>>> is_numeric_dtype(np.timedelta64)
False
- >>> is_numeric_dtype(np.array(['a', 'b']))
+ >>> is_numeric_dtype(np.array(["a", "b"]))
False
>>> is_numeric_dtype(pd.Series([1, 2]))
True
- >>> is_numeric_dtype(pd.Index([1, 2.]))
+ >>> is_numeric_dtype(pd.Index([1, 2.0]))
True
>>> is_numeric_dtype(np.array([], dtype=np.timedelta64))
False
@@ -1168,11 +1187,11 @@ def is_float_dtype(arr_or_dtype) -> bool:
False
>>> is_float_dtype(float)
True
- >>> is_float_dtype(np.array(['a', 'b']))
+ >>> is_float_dtype(np.array(["a", "b"]))
False
>>> is_float_dtype(pd.Series([1, 2]))
False
- >>> is_float_dtype(pd.Index([1, 2.]))
+ >>> is_float_dtype(pd.Index([1, 2.0]))
True
"""
return _is_dtype_type(arr_or_dtype, classes(np.floating)) or _is_dtype(
@@ -1210,7 +1229,7 @@ def is_bool_dtype(arr_or_dtype) -> bool:
True
>>> is_bool_dtype(np.bool_)
True
- >>> is_bool_dtype(np.array(['a', 'b']))
+ >>> is_bool_dtype(np.array(["a", "b"]))
False
>>> is_bool_dtype(pd.Series([1, 2]))
False
@@ -1294,13 +1313,13 @@ def is_extension_array_dtype(arr_or_dtype) -> bool:
Examples
--------
>>> from pandas.api.types import is_extension_array_dtype
- >>> arr = pd.Categorical(['a', 'b'])
+ >>> arr = pd.Categorical(["a", "b"])
>>> is_extension_array_dtype(arr)
True
>>> is_extension_array_dtype(arr.dtype)
True
- >>> arr = np.array(['a', 'b'])
+ >>> arr = np.array(["a", "b"])
>>> is_extension_array_dtype(arr.dtype)
False
"""
@@ -1347,7 +1366,7 @@ def is_complex_dtype(arr_or_dtype) -> bool:
False
>>> is_complex_dtype(np.complex128)
True
- >>> is_complex_dtype(np.array(['a', 'b']))
+ >>> is_complex_dtype(np.array(["a", "b"]))
False
>>> is_complex_dtype(pd.Series([1, 2]))
False
@@ -1548,8 +1567,8 @@ def _validate_date_like_dtype(dtype) -> None:
raise TypeError(e) from e
if typ not in ["generic", "ns"]:
raise ValueError(
- f"{repr(dtype.name)} is too specific of a frequency, "
- f"try passing {repr(dtype.type.__name__)}"
+ f"{dtype.name!r} is too specific of a frequency, "
+ f"try passing {dtype.type.__name__!r}"
)
@@ -1706,7 +1725,6 @@ def is_all_strings(value: ArrayLike) -> bool:
"is_float_dtype",
"is_int64_dtype",
"is_integer_dtype",
- "is_interval",
"is_interval_dtype",
"is_iterator",
"is_named_tuple",
diff --git a/pandas/core/dtypes/concat.py b/pandas/core/dtypes/concat.py
index 9ec662a6cd352..17e68b0e19a68 100644
--- a/pandas/core/dtypes/concat.py
+++ b/pandas/core/dtypes/concat.py
@@ -1,18 +1,17 @@
"""
Utility functions related to concat.
"""
+
from __future__ import annotations
from typing import (
TYPE_CHECKING,
cast,
)
-import warnings
import numpy as np
from pandas._libs import lib
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.astype import astype_array
from pandas.core.dtypes.cast import (
@@ -41,7 +40,7 @@
)
-def _is_nonempty(x, axis) -> bool:
+def _is_nonempty(x: ArrayLike, axis: AxisInt) -> bool:
# filter empty arrays
# 1-d dtypes always are included here
if x.ndim <= axis:
@@ -100,28 +99,10 @@ def concat_compat(
# Creating an empty array directly is tempting, but the winnings would be
# marginal given that it would still require shape & dtype calculation and
# np.concatenate which has them both implemented is compiled.
- orig = to_concat
non_empties = [x for x in to_concat if _is_nonempty(x, axis)]
- if non_empties and axis == 0 and not ea_compat_axis:
- # ea_compat_axis see GH#39574
- to_concat = non_empties
any_ea, kinds, target_dtype = _get_result_dtype(to_concat, non_empties)
- if len(to_concat) < len(orig):
- _, _, alt_dtype = _get_result_dtype(orig, non_empties)
- if alt_dtype != target_dtype:
- # GH#39122
- warnings.warn(
- "The behavior of array concatenation with empty entries is "
- "deprecated. In a future version, this will no longer exclude "
- "empty items when determining the result dtype. "
- "To retain the old behavior, exclude the empty entries before "
- "the concat operation.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
if target_dtype is not None:
to_concat = [astype_array(arr, target_dtype, copy=False) for arr in to_concat]
@@ -278,8 +259,8 @@ def union_categoricals(
containing categorical data, but note that the resulting array will
always be a plain `Categorical`
- >>> a = pd.Series(["b", "c"], dtype='category')
- >>> b = pd.Series(["a", "b"], dtype='category')
+ >>> a = pd.Series(["b", "c"], dtype="category")
+ >>> b = pd.Series(["a", "b"], dtype="category")
>>> pd.api.types.union_categoricals([a, b])
['b', 'c', 'a', 'b']
Categories (3, object): ['b', 'c', 'a']
diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py
index ed5256922377a..f94d32a3b8547 100644
--- a/pandas/core/dtypes/dtypes.py
+++ b/pandas/core/dtypes/dtypes.py
@@ -1,6 +1,7 @@
"""
Define extension dtypes.
"""
+
from __future__ import annotations
from datetime import (
@@ -21,6 +22,8 @@
import numpy as np
import pytz
+from pandas._config.config import get_option
+
from pandas._libs import (
lib,
missing as libmissing,
@@ -62,8 +65,6 @@
is_list_like,
)
-from pandas.util import capitalize_first_letter
-
if not pa_version_under10p1:
import pyarrow as pa
@@ -190,8 +191,8 @@ class CategoricalDtype(PandasExtensionDtype, ExtensionDtype):
Examples
--------
- >>> t = pd.CategoricalDtype(categories=['b', 'a'], ordered=True)
- >>> pd.Series(['a', 'b', 'a', 'c'], dtype=t)
+ >>> t = pd.CategoricalDtype(categories=["b", "a"], ordered=True)
+ >>> pd.Series(["a", "b", "a", "c"], dtype=t)
0 a
1 b
2 a
@@ -288,14 +289,14 @@ def _from_values_or_dtype(
>>> pd.CategoricalDtype._from_values_or_dtype()
CategoricalDtype(categories=None, ordered=None, categories_dtype=None)
>>> pd.CategoricalDtype._from_values_or_dtype(
- ... categories=['a', 'b'], ordered=True
+ ... categories=["a", "b"], ordered=True
... )
CategoricalDtype(categories=['a', 'b'], ordered=True, categories_dtype=object)
- >>> dtype1 = pd.CategoricalDtype(['a', 'b'], ordered=True)
- >>> dtype2 = pd.CategoricalDtype(['x', 'y'], ordered=False)
+ >>> dtype1 = pd.CategoricalDtype(["a", "b"], ordered=True)
+ >>> dtype2 = pd.CategoricalDtype(["x", "y"], ordered=False)
>>> c = pd.Categorical([0, 1], dtype=dtype1)
>>> pd.CategoricalDtype._from_values_or_dtype(
- ... c, ['x', 'y'], ordered=True, dtype=dtype2
+ ... c, ["x", "y"], ordered=True, dtype=dtype2
... )
Traceback (most recent call last):
...
@@ -318,7 +319,7 @@ def _from_values_or_dtype(
dtype = CategoricalDtype(categories, ordered)
else:
- raise ValueError(f"Unknown dtype {repr(dtype)}")
+ raise ValueError(f"Unknown dtype {dtype!r}")
elif categories is not None or ordered is not None:
raise ValueError(
"Cannot specify `categories` or `ordered` together with `dtype`."
@@ -566,7 +567,7 @@ def validate_categories(categories, fastpath: bool = False) -> Index:
if not fastpath and not is_list_like(categories):
raise TypeError(
- f"Parameter 'categories' must be list-like, was {repr(categories)}"
+ f"Parameter 'categories' must be list-like, was {categories!r}"
)
if not isinstance(categories, ABCIndex):
categories = Index._with_infer(categories, tupleize_cols=False)
@@ -602,7 +603,7 @@ def update_dtype(self, dtype: str_type | CategoricalDtype) -> CategoricalDtype:
elif not self.is_dtype(dtype):
raise ValueError(
f"a CategoricalDtype must be passed to perform an update, "
- f"got {repr(dtype)}"
+ f"got {dtype!r}"
)
else:
# from here on, dtype is a CategoricalDtype
@@ -623,7 +624,7 @@ def categories(self) -> Index:
Examples
--------
- >>> cat_type = pd.CategoricalDtype(categories=['a', 'b'], ordered=True)
+ >>> cat_type = pd.CategoricalDtype(categories=["a", "b"], ordered=True)
>>> cat_type.categories
Index(['a', 'b'], dtype='object')
"""
@@ -636,11 +637,11 @@ def ordered(self) -> Ordered:
Examples
--------
- >>> cat_type = pd.CategoricalDtype(categories=['a', 'b'], ordered=True)
+ >>> cat_type = pd.CategoricalDtype(categories=["a", "b"], ordered=True)
>>> cat_type.ordered
True
- >>> cat_type = pd.CategoricalDtype(categories=['a', 'b'], ordered=False)
+ >>> cat_type = pd.CategoricalDtype(categories=["a", "b"], ordered=False)
>>> cat_type.ordered
False
"""
@@ -719,10 +720,10 @@ class DatetimeTZDtype(PandasExtensionDtype):
Examples
--------
>>> from zoneinfo import ZoneInfo
- >>> pd.DatetimeTZDtype(tz=ZoneInfo('UTC'))
+ >>> pd.DatetimeTZDtype(tz=ZoneInfo("UTC"))
datetime64[ns, UTC]
- >>> pd.DatetimeTZDtype(tz=ZoneInfo('Europe/Paris'))
+ >>> pd.DatetimeTZDtype(tz=ZoneInfo("Europe/Paris"))
datetime64[ns, Europe/Paris]
"""
@@ -795,7 +796,7 @@ def unit(self) -> str_type:
Examples
--------
>>> from zoneinfo import ZoneInfo
- >>> dtype = pd.DatetimeTZDtype(tz=ZoneInfo('America/Los_Angeles'))
+ >>> dtype = pd.DatetimeTZDtype(tz=ZoneInfo("America/Los_Angeles"))
>>> dtype.unit
'ns'
"""
@@ -809,7 +810,7 @@ def tz(self) -> tzinfo:
Examples
--------
>>> from zoneinfo import ZoneInfo
- >>> dtype = pd.DatetimeTZDtype(tz=ZoneInfo('America/Los_Angeles'))
+ >>> dtype = pd.DatetimeTZDtype(tz=ZoneInfo("America/Los_Angeles"))
>>> dtype.tz
zoneinfo.ZoneInfo(key='America/Los_Angeles')
"""
@@ -842,7 +843,7 @@ def construct_from_string(cls, string: str_type) -> DatetimeTZDtype:
Examples
--------
- >>> DatetimeTZDtype.construct_from_string('datetime64[ns, UTC]')
+ >>> DatetimeTZDtype.construct_from_string("datetime64[ns, UTC]")
datetime64[ns, UTC]
"""
if not isinstance(string, str):
@@ -919,7 +920,7 @@ def __from_arrow__(self, array: pa.Array | pa.ChunkedArray) -> DatetimeArray:
else:
np_arr = array.to_numpy()
- return DatetimeArray._from_sequence(np_arr, dtype=self, copy=False)
+ return DatetimeArray._simple_new(np_arr, dtype=self)
def __setstate__(self, state) -> None:
# for pickle compat. __get_state__ is defined in the
@@ -964,7 +965,7 @@ class PeriodDtype(PeriodDtypeBase, PandasExtensionDtype):
Examples
--------
- >>> pd.PeriodDtype(freq='D')
+ >>> pd.PeriodDtype(freq="D")
period[D]
>>> pd.PeriodDtype(freq=pd.offsets.MonthEnd())
@@ -1028,7 +1029,7 @@ def freq(self) -> BaseOffset:
Examples
--------
- >>> dtype = pd.PeriodDtype(freq='D')
+ >>> dtype = pd.PeriodDtype(freq="D")
>>> dtype.freq
"""
@@ -1087,7 +1088,7 @@ def na_value(self) -> NaTType:
def __eq__(self, other: object) -> bool:
if isinstance(other, str):
- return other in [self.name, capitalize_first_letter(self.name)]
+ return other[:1].lower() + other[1:] == self.name
return super().__eq__(other)
@@ -1183,7 +1184,7 @@ class IntervalDtype(PandasExtensionDtype):
Examples
--------
- >>> pd.IntervalDtype(subtype='int64', closed='both')
+ >>> pd.IntervalDtype(subtype="int64", closed="both")
interval[int64, both]
"""
@@ -1283,7 +1284,7 @@ def subtype(self):
Examples
--------
- >>> dtype = pd.IntervalDtype(subtype='int64', closed='both')
+ >>> dtype = pd.IntervalDtype(subtype="int64", closed="both")
>>> dtype.subtype
dtype('int64')
"""
@@ -1458,7 +1459,7 @@ def __init__(self, dtype: npt.DTypeLike | NumpyEADtype | None) -> None:
self._dtype = np.dtype(dtype)
def __repr__(self) -> str:
- return f"NumpyEADtype({repr(self.name)})"
+ return f"NumpyEADtype({self.name!r})"
@property
def numpy_dtype(self) -> np.dtype:
@@ -1704,17 +1705,15 @@ def __eq__(self, other: object) -> bool:
if isinstance(other, type(self)):
subtype = self.subtype == other.subtype
- if self._is_na_fill_value:
+ if self._is_na_fill_value or other._is_na_fill_value:
# this case is complicated by two things:
# SparseDtype(float, float(nan)) == SparseDtype(float, np.nan)
# SparseDtype(float, np.nan) != SparseDtype(float, pd.NaT)
# i.e. we want to treat any floating-point NaN as equal, but
# not a floating-point NaN and a datetime NaT.
- fill_value = (
- other._is_na_fill_value
- and isinstance(self.fill_value, type(other.fill_value))
- or isinstance(other.fill_value, type(self.fill_value))
- )
+ fill_value = isinstance(
+ self.fill_value, type(other.fill_value)
+ ) or isinstance(other.fill_value, type(self.fill_value))
else:
with warnings.catch_warnings():
# Ignore spurious numpy warning
@@ -1814,7 +1813,7 @@ def subtype(self):
@property
def name(self) -> str:
- return f"Sparse[{self.subtype.name}, {repr(self.fill_value)}]"
+ return f"Sparse[{self.subtype.name}, {self.fill_value!r}]"
def __repr__(self) -> str:
return self.name
@@ -2001,7 +2000,7 @@ def _subtype_with_str(self):
>>> SparseDtype(object, 1)._subtype_with_str
dtype('O')
- >>> dtype = SparseDtype(str, '')
+ >>> dtype = SparseDtype(str, "")
>>> dtype.subtype
dtype('O')
@@ -2030,7 +2029,9 @@ def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None:
# np.nan isn't a singleton, so we may end up with multiple
# NaNs here, so we ignore the all NA case too.
- if not (len(set(fill_values)) == 1 or isna(fill_values).all()):
+ if get_option("performance_warnings") and (
+ not (len(set(fill_values)) == 1 or isna(fill_values).all())
+ ):
warnings.warn(
"Concatenating sparse arrays with multiple fill "
f"values: '{fill_values}'. Picking the first and "
@@ -2074,6 +2075,10 @@ class ArrowDtype(StorageExtensionDtype):
-------
ArrowDtype
+ See Also
+ --------
+ DataFrame.convert_dtypes : Convert columns to the best possible dtypes.
+
Examples
--------
>>> import pyarrow as pa
@@ -2173,7 +2178,7 @@ def name(self) -> str: # type: ignore[override]
"""
A string identifying the data type.
"""
- return f"{str(self.pyarrow_dtype)}[{self.storage}]"
+ return f"{self.pyarrow_dtype!s}[{self.storage}]"
@cache_readonly
def numpy_dtype(self) -> np.dtype:
@@ -2190,7 +2195,9 @@ def numpy_dtype(self) -> np.dtype:
# This can be removed if/when pyarrow addresses it:
# https://github.com/apache/arrow/issues/34462
return np.dtype(f"timedelta64[{self.pyarrow_dtype.unit}]")
- if pa.types.is_string(self.pyarrow_dtype):
+ if pa.types.is_string(self.pyarrow_dtype) or pa.types.is_large_string(
+ self.pyarrow_dtype
+ ):
# pa.string().to_pandas_dtype() = object which we don't want
return np.dtype(str)
try:
@@ -2337,7 +2344,7 @@ def _get_common_dtype(self, dtypes: list[DtypeObj]) -> DtypeObj | None:
except NotImplementedError:
return None
- def __from_arrow__(self, array: pa.Array | pa.ChunkedArray):
+ def __from_arrow__(self, array: pa.Array | pa.ChunkedArray) -> ArrowExtensionArray:
"""
Construct IntegerArray/FloatingArray from pyarrow Array/ChunkedArray.
"""
diff --git a/pandas/core/dtypes/generic.py b/pandas/core/dtypes/generic.py
index 9718ad600cb80..8d3d86217dedf 100644
--- a/pandas/core/dtypes/generic.py
+++ b/pandas/core/dtypes/generic.py
@@ -1,4 +1,5 @@
-""" define generic base classes for pandas objects """
+"""define generic base classes for pandas objects"""
+
from __future__ import annotations
from typing import (
@@ -33,7 +34,7 @@
# define abstract base classes to enable isinstance type checking on our
# objects
-def create_pandas_abc_type(name, attr, comp):
+def create_pandas_abc_type(name, attr, comp) -> type:
def _check(inst) -> bool:
return getattr(inst, attr, "_typ") in comp
diff --git a/pandas/core/dtypes/inference.py b/pandas/core/dtypes/inference.py
index e87b7f02b9b05..f042911b53d2b 100644
--- a/pandas/core/dtypes/inference.py
+++ b/pandas/core/dtypes/inference.py
@@ -1,4 +1,4 @@
-""" basic inference routines """
+"""basic inference routines"""
from __future__ import annotations
@@ -29,8 +29,6 @@
is_decimal = lib.is_decimal
-is_interval = lib.is_interval
-
is_list_like = lib.is_list_like
is_iterator = lib.is_iterator
@@ -266,7 +264,7 @@ def is_nested_list_like(obj: object) -> bool:
is_list_like(obj)
and hasattr(obj, "__len__")
# need PEP 724 to handle these typing errors
- and len(obj) > 0 # pyright: ignore[reportGeneralTypeIssues]
+ and len(obj) > 0 # pyright: ignore[reportArgumentType]
and all(is_list_like(item) for item in obj) # type: ignore[attr-defined]
)
@@ -428,7 +426,7 @@ def is_dataclass(item: object) -> bool:
>>> is_dataclass(Point)
False
- >>> is_dataclass(Point(0,2))
+ >>> is_dataclass(Point(0, 2))
True
"""
diff --git a/pandas/core/dtypes/missing.py b/pandas/core/dtypes/missing.py
index 3e4227a8a2598..f127c736e745a 100644
--- a/pandas/core/dtypes/missing.py
+++ b/pandas/core/dtypes/missing.py
@@ -1,10 +1,10 @@
"""
missing types & inference
"""
+
from __future__ import annotations
from decimal import Decimal
-from functools import partial
from typing import (
TYPE_CHECKING,
overload,
@@ -13,8 +13,6 @@
import numpy as np
-from pandas._config import get_option
-
from pandas._libs import lib
import pandas._libs.missing as libmissing
from pandas._libs.tslibs import (
@@ -48,6 +46,8 @@
if TYPE_CHECKING:
from re import Pattern
+ from pandas._libs.missing import NAType
+ from pandas._libs.tslibs import NaTType
from pandas._typing import (
ArrayLike,
DtypeObj,
@@ -64,38 +64,33 @@
isposinf_scalar = libmissing.isposinf_scalar
isneginf_scalar = libmissing.isneginf_scalar
-nan_checker = np.isnan
-INF_AS_NA = False
_dtype_object = np.dtype("object")
_dtype_str = np.dtype(str)
@overload
-def isna(obj: Scalar | Pattern) -> bool:
- ...
+def isna(obj: Scalar | Pattern | NAType | NaTType) -> bool: ...
@overload
def isna(
obj: ArrayLike | Index | list,
-) -> npt.NDArray[np.bool_]:
- ...
+) -> npt.NDArray[np.bool_]: ...
@overload
-def isna(obj: NDFrameT) -> NDFrameT:
- ...
+def isna(obj: NDFrameT) -> NDFrameT: ...
# handle unions
@overload
-def isna(obj: NDFrameT | ArrayLike | Index | list) -> NDFrameT | npt.NDArray[np.bool_]:
- ...
+def isna(
+ obj: NDFrameT | ArrayLike | Index | list,
+) -> NDFrameT | npt.NDArray[np.bool_]: ...
@overload
-def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
- ...
+def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame: ...
def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
@@ -129,7 +124,7 @@ def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
--------
Scalar arguments (including strings) result in a scalar boolean.
- >>> pd.isna('dog')
+ >>> pd.isna("dog")
False
>>> pd.isna(pd.NA)
@@ -150,8 +145,7 @@ def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
For indexes, an ndarray of booleans is returned.
- >>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None,
- ... "2017-07-08"])
+ >>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None, "2017-07-08"])
>>> index
DatetimeIndex(['2017-07-05', '2017-07-06', 'NaT', '2017-07-08'],
dtype='datetime64[ns]', freq=None)
@@ -160,7 +154,7 @@ def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
For Series and DataFrame, the same type is returned, containing booleans.
- >>> df = pd.DataFrame([['ant', 'bee', 'cat'], ['dog', None, 'fly']])
+ >>> df = pd.DataFrame([["ant", "bee", "cat"], ["dog", None, "fly"]])
>>> df
0 1 2
0 ant bee cat
@@ -181,84 +175,50 @@ def isna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
isnull = isna
-def _isna(obj, inf_as_na: bool = False):
+def _isna(obj):
"""
- Detect missing values, treating None, NaN or NA as null. Infinite
- values will also be treated as null if inf_as_na is True.
+ Detect missing values, treating None, NaN or NA as null.
Parameters
----------
obj: ndarray or object value
Input array or scalar value.
- inf_as_na: bool
- Whether to treat infinity as null.
Returns
-------
boolean ndarray or boolean
"""
if is_scalar(obj):
- return libmissing.checknull(obj, inf_as_na=inf_as_na)
+ return libmissing.checknull(obj)
elif isinstance(obj, ABCMultiIndex):
raise NotImplementedError("isna is not defined for MultiIndex")
elif isinstance(obj, type):
return False
elif isinstance(obj, (np.ndarray, ABCExtensionArray)):
- return _isna_array(obj, inf_as_na=inf_as_na)
+ return _isna_array(obj)
elif isinstance(obj, ABCIndex):
# Try to use cached isna, which also short-circuits for integer dtypes
# and avoids materializing RangeIndex._values
if not obj._can_hold_na:
return obj.isna()
- return _isna_array(obj._values, inf_as_na=inf_as_na)
+ return _isna_array(obj._values)
elif isinstance(obj, ABCSeries):
- result = _isna_array(obj._values, inf_as_na=inf_as_na)
+ result = _isna_array(obj._values)
# box
result = obj._constructor(result, index=obj.index, name=obj.name, copy=False)
return result
elif isinstance(obj, ABCDataFrame):
return obj.isna()
elif isinstance(obj, list):
- return _isna_array(np.asarray(obj, dtype=object), inf_as_na=inf_as_na)
+ return _isna_array(np.asarray(obj, dtype=object))
elif hasattr(obj, "__array__"):
- return _isna_array(np.asarray(obj), inf_as_na=inf_as_na)
+ return _isna_array(np.asarray(obj))
else:
return False
-def _use_inf_as_na(key) -> None:
- """
- Option change callback for na/inf behaviour.
-
- Choose which replacement for numpy.isnan / -numpy.isfinite is used.
-
- Parameters
- ----------
- flag: bool
- True means treat None, NaN, INF, -INF as null (old way),
- False means None and NaN are null, but INF, -INF are not null
- (new way).
-
- Notes
- -----
- This approach to setting global module values is discussed and
- approved here:
-
- * https://stackoverflow.com/questions/4859217/
- programmatically-creating-variables-in-python/4859312#4859312
- """
- inf_as_na = get_option(key)
- globals()["_isna"] = partial(_isna, inf_as_na=inf_as_na)
- if inf_as_na:
- globals()["nan_checker"] = lambda x: ~np.isfinite(x)
- globals()["INF_AS_NA"] = True
- else:
- globals()["nan_checker"] = np.isnan
- globals()["INF_AS_NA"] = False
-
-
-def _isna_array(values: ArrayLike, inf_as_na: bool = False):
+def _isna_array(values: ArrayLike) -> npt.NDArray[np.bool_] | NDFrame:
"""
Return an array indicating which values of the input array are NaN / NA.
@@ -266,8 +226,6 @@ def _isna_array(values: ArrayLike, inf_as_na: bool = False):
----------
obj: ndarray or ExtensionArray
The input array whose elements are to be checked.
- inf_as_na: bool
- Whether or not to treat infinite values as NA.
Returns
-------
@@ -275,34 +233,29 @@ def _isna_array(values: ArrayLike, inf_as_na: bool = False):
Array of boolean values denoting the NA status of each element.
"""
dtype = values.dtype
+ result: npt.NDArray[np.bool_] | NDFrame
if not isinstance(values, np.ndarray):
# i.e. ExtensionArray
- if inf_as_na and isinstance(dtype, CategoricalDtype):
- result = libmissing.isnaobj(values.to_numpy(), inf_as_na=inf_as_na)
- else:
- # error: Incompatible types in assignment (expression has type
- # "Union[ndarray[Any, Any], ExtensionArraySupportsAnyAll]", variable has
- # type "ndarray[Any, dtype[bool_]]")
- result = values.isna() # type: ignore[assignment]
+ # error: Incompatible types in assignment (expression has type
+ # "Union[ndarray[Any, Any], ExtensionArraySupportsAnyAll]", variable has
+ # type "ndarray[Any, dtype[bool_]]")
+ result = values.isna() # type: ignore[assignment]
elif isinstance(values, np.rec.recarray):
# GH 48526
- result = _isna_recarray_dtype(values, inf_as_na=inf_as_na)
+ result = _isna_recarray_dtype(values)
elif is_string_or_object_np_dtype(values.dtype):
- result = _isna_string_dtype(values, inf_as_na=inf_as_na)
+ result = _isna_string_dtype(values)
elif dtype.kind in "mM":
# this is the NaT pattern
result = values.view("i8") == iNaT
else:
- if inf_as_na:
- result = ~np.isfinite(values)
- else:
- result = np.isnan(values)
+ result = np.isnan(values)
return result
-def _isna_string_dtype(values: np.ndarray, inf_as_na: bool) -> npt.NDArray[np.bool_]:
+def _isna_string_dtype(values: np.ndarray) -> npt.NDArray[np.bool_]:
# Working around NumPy ticket 1542
dtype = values.dtype
@@ -310,71 +263,48 @@ def _isna_string_dtype(values: np.ndarray, inf_as_na: bool) -> npt.NDArray[np.bo
result = np.zeros(values.shape, dtype=bool)
else:
if values.ndim in {1, 2}:
- result = libmissing.isnaobj(values, inf_as_na=inf_as_na)
+ result = libmissing.isnaobj(values)
else:
# 0-D, reached via e.g. mask_missing
- result = libmissing.isnaobj(values.ravel(), inf_as_na=inf_as_na)
+ result = libmissing.isnaobj(values.ravel())
result = result.reshape(values.shape)
return result
-def _has_record_inf_value(record_as_array: np.ndarray) -> np.bool_:
- is_inf_in_record = np.zeros(len(record_as_array), dtype=bool)
- for i, value in enumerate(record_as_array):
- is_element_inf = False
- try:
- is_element_inf = np.isinf(value)
- except TypeError:
- is_element_inf = False
- is_inf_in_record[i] = is_element_inf
-
- return np.any(is_inf_in_record)
-
-
-def _isna_recarray_dtype(
- values: np.rec.recarray, inf_as_na: bool
-) -> npt.NDArray[np.bool_]:
+def _isna_recarray_dtype(values: np.rec.recarray) -> npt.NDArray[np.bool_]:
result = np.zeros(values.shape, dtype=bool)
for i, record in enumerate(values):
record_as_array = np.array(record.tolist())
does_record_contain_nan = isna_all(record_as_array)
- does_record_contain_inf = False
- if inf_as_na:
- does_record_contain_inf = bool(_has_record_inf_value(record_as_array))
- result[i] = np.any(
- np.logical_or(does_record_contain_nan, does_record_contain_inf)
- )
+ result[i] = np.any(does_record_contain_nan)
return result
@overload
-def notna(obj: Scalar) -> bool:
- ...
+def notna(obj: Scalar | Pattern | NAType | NaTType) -> bool: ...
@overload
def notna(
obj: ArrayLike | Index | list,
-) -> npt.NDArray[np.bool_]:
- ...
+) -> npt.NDArray[np.bool_]: ...
@overload
-def notna(obj: NDFrameT) -> NDFrameT:
- ...
+def notna(obj: NDFrameT) -> NDFrameT: ...
# handle unions
@overload
-def notna(obj: NDFrameT | ArrayLike | Index | list) -> NDFrameT | npt.NDArray[np.bool_]:
- ...
+def notna(
+ obj: NDFrameT | ArrayLike | Index | list,
+) -> NDFrameT | npt.NDArray[np.bool_]: ...
@overload
-def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
- ...
+def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame: ...
def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
@@ -408,7 +338,7 @@ def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
--------
Scalar arguments (including strings) result in a scalar boolean.
- >>> pd.notna('dog')
+ >>> pd.notna("dog")
True
>>> pd.notna(pd.NA)
@@ -429,8 +359,7 @@ def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
For indexes, an ndarray of booleans is returned.
- >>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None,
- ... "2017-07-08"])
+ >>> index = pd.DatetimeIndex(["2017-07-05", "2017-07-06", None, "2017-07-08"])
>>> index
DatetimeIndex(['2017-07-05', '2017-07-06', 'NaT', '2017-07-08'],
dtype='datetime64[ns]', freq=None)
@@ -439,7 +368,7 @@ def notna(obj: object) -> bool | npt.NDArray[np.bool_] | NDFrame:
For Series and DataFrame, the same type is returned, containing booleans.
- >>> df = pd.DataFrame([['ant', 'bee', 'cat'], ['dog', None, 'fly']])
+ >>> df = pd.DataFrame([["ant", "bee", "cat"], ["dog", None, "fly"]])
>>> df
0 1 2
0 ant bee cat
@@ -495,13 +424,9 @@ def array_equivalent(
Examples
--------
- >>> array_equivalent(
- ... np.array([1, 2, np.nan]),
- ... np.array([1, 2, np.nan]))
+ >>> array_equivalent(np.array([1, 2, np.nan]), np.array([1, 2, np.nan]))
True
- >>> array_equivalent(
- ... np.array([1, np.nan, 2]),
- ... np.array([1, 2, np.nan]))
+ >>> array_equivalent(np.array([1, np.nan, 2]), np.array([1, 2, np.nan]))
False
"""
left, right = np.asarray(left), np.asarray(right)
@@ -634,7 +559,7 @@ def infer_fill_value(val):
"""
if not is_list_like(val):
val = [val]
- val = np.array(val, copy=False)
+ val = np.asarray(val)
if val.dtype.kind in "mM":
return np.array("NaT", dtype=val.dtype)
elif val.dtype == object:
@@ -649,6 +574,20 @@ def infer_fill_value(val):
return np.nan
+def construct_1d_array_from_inferred_fill_value(
+ value: object, length: int
+) -> ArrayLike:
+ # Find our empty_value dtype by constructing an array
+ # from our value and doing a .take on it
+ from pandas.core.algorithms import take_nd
+ from pandas.core.construction import sanitize_array
+ from pandas.core.indexes.base import Index
+
+ arr = sanitize_array(value, Index(range(1)), copy=False)
+ taker = -1 * np.ones(length, dtype=np.intp)
+ return take_nd(arr, taker)
+
+
def maybe_fill(arr: np.ndarray) -> np.ndarray:
"""
Fill numpy.ndarray with NaN, unless we have a integer or boolean dtype.
@@ -673,15 +612,15 @@ def na_value_for_dtype(dtype: DtypeObj, compat: bool = True):
Examples
--------
- >>> na_value_for_dtype(np.dtype('int64'))
+ >>> na_value_for_dtype(np.dtype("int64"))
0
- >>> na_value_for_dtype(np.dtype('int64'), compat=False)
+ >>> na_value_for_dtype(np.dtype("int64"), compat=False)
nan
- >>> na_value_for_dtype(np.dtype('float64'))
+ >>> na_value_for_dtype(np.dtype("float64"))
nan
- >>> na_value_for_dtype(np.dtype('bool'))
+ >>> na_value_for_dtype(np.dtype("bool"))
False
- >>> na_value_for_dtype(np.dtype('datetime64[ns]'))
+ >>> na_value_for_dtype(np.dtype("datetime64[ns]"))
numpy.datetime64('NaT')
"""
@@ -777,7 +716,7 @@ def isna_all(arr: ArrayLike) -> bool:
dtype = arr.dtype
if lib.is_np_dtype(dtype, "f"):
- checker = nan_checker
+ checker = np.isnan
elif (lib.is_np_dtype(dtype, "mM")) or isinstance(
dtype, (DatetimeTZDtype, PeriodDtype)
@@ -789,9 +728,7 @@ def isna_all(arr: ArrayLike) -> bool:
else:
# error: Incompatible types in assignment (expression has type "Callable[[Any],
# Any]", variable has type "ufunc")
- checker = lambda x: _isna_array( # type: ignore[assignment]
- x, inf_as_na=INF_AS_NA
- )
+ checker = _isna_array # type: ignore[assignment]
return all(
checker(arr[i : i + chunk_len]).all() for i in range(0, total_len, chunk_len)
diff --git a/pandas/core/flags.py b/pandas/core/flags.py
index 394695e69a3d3..eceb86dc61d9f 100644
--- a/pandas/core/flags.py
+++ b/pandas/core/flags.py
@@ -11,6 +11,10 @@ class Flags:
"""
Flags that apply to pandas objects.
+ “Flags” differ from “metadata”. Flags reflect properties of the pandas
+ object (the Series or DataFrame). Metadata refer to properties of the
+ dataset, and should be stored in DataFrame.attrs.
+
Parameters
----------
obj : Series or DataFrame
@@ -30,6 +34,11 @@ class Flags:
it is expected that every method taking or returning one or more
DataFrame or Series objects will propagate ``allows_duplicate_labels``.
+ See Also
+ --------
+ DataFrame.attrs : Dictionary of global attributes of this dataset.
+ Series.attrs : Dictionary of global attributes of this dataset.
+
Examples
--------
Attributes can be set in two ways:
@@ -41,7 +50,7 @@ class Flags:
>>> df.flags
- >>> df.flags['allows_duplicate_labels'] = True
+ >>> df.flags["allows_duplicate_labels"] = True
>>> df.flags
"""
@@ -71,7 +80,7 @@ def allows_duplicate_labels(self) -> bool:
Examples
--------
- >>> df = pd.DataFrame({"A": [1, 2]}, index=['a', 'a'])
+ >>> df = pd.DataFrame({"A": [1, 2]}, index=["a", "a"])
>>> df.flags.allows_duplicate_labels
True
>>> df.flags.allows_duplicate_labels = False
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
index 73b5804d8c168..50a93994dc76b 100644
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -8,6 +8,7 @@
alignment and a host of useful data manipulation methods having to do with the
labeling information
"""
+
from __future__ import annotations
import collections
@@ -39,12 +40,7 @@
import numpy as np
from numpy import ma
-from pandas._config import (
- get_option,
- using_copy_on_write,
- warn_copy_on_write,
-)
-from pandas._config.config import _get_option
+from pandas._config import get_option
from pandas._libs import (
algos as libalgos,
@@ -60,16 +56,16 @@
from pandas.errors import (
ChainedAssignmentError,
InvalidIndexError,
+)
+from pandas.errors.cow import (
_chained_assignment_method_msg,
_chained_assignment_msg,
- _chained_assignment_warning_method_msg,
- _chained_assignment_warning_msg,
)
from pandas.util._decorators import (
Appender,
Substitution,
- deprecate_nonkeyword_arguments,
doc,
+ set_module,
)
from pandas.util._exceptions import (
find_stack_level,
@@ -89,7 +85,6 @@
find_common_type,
infer_dtype_from_scalar,
invalidate_string_dtypes,
- maybe_box_native,
maybe_downcast_to_dtype,
)
from pandas.core.dtypes.common import (
@@ -167,15 +162,11 @@
check_bool_indexer,
check_dict_or_set_indexers,
)
-from pandas.core.internals import (
- ArrayManager,
- BlockManager,
-)
+from pandas.core.internals import BlockManager
from pandas.core.internals.construction import (
arrays_to_mgr,
dataclasses_to_dicts,
dict_to_mgr,
- mgr_to_mgr,
ndarray_to_mgr,
nested_data_to_arrays,
rec_array_to_mgr,
@@ -228,6 +219,8 @@
FormattersType,
Frequency,
FromDictOrient,
+ HashableT,
+ HashableT2,
IgnoreRaise,
IndexKeyFunc,
IndexLabel,
@@ -237,7 +230,6 @@
MergeHow,
MergeValidate,
MutableMappingT,
- NaAction,
NaPosition,
NsmallestNlargestKeep,
PythonFuncType,
@@ -251,7 +243,7 @@
SortKind,
StorageOptions,
Suffixes,
- ToGbqIfexist,
+ T,
ToStataByteorder,
ToTimestampHow,
UpdateJoin,
@@ -263,7 +255,7 @@
from pandas.core.groupby.generic import DataFrameGroupBy
from pandas.core.interchange.dataframe_protocol import DataFrame as DataFrameXchg
- from pandas.core.internals import SingleDataManager
+ from pandas.core.internals.managers import SingleBlockManager
from pandas.io.formats.style import Styler
@@ -365,7 +357,7 @@
of a string to indicate that the column name from `left` or
`right` should be left as-is, with no suffix. At least one of the
values must not be None.
-copy : bool, default True
+copy : bool, default False
If False, avoid copy if possible.
.. note::
@@ -379,6 +371,8 @@
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+
+ .. deprecated:: 3.0.0
indicator : bool or str, default False
If True, adds a column to the output DataFrame called "_merge" with
information on the source of each row. The column can be given a different
@@ -507,6 +501,7 @@
# DataFrame class
+@set_module("pandas")
class DataFrame(NDFrame, OpsMixin):
"""
Two-dimensional, size-mutable, potentially heterogeneous tabular data.
@@ -561,7 +556,7 @@ class DataFrame(NDFrame, OpsMixin):
--------
Constructing DataFrame from a dictionary.
- >>> d = {'col1': [1, 2], 'col2': [3, 4]}
+ >>> d = {"col1": [1, 2], "col2": [3, 4]}
>>> df = pd.DataFrame(data=d)
>>> df
col1 col2
@@ -585,7 +580,7 @@ class DataFrame(NDFrame, OpsMixin):
Constructing DataFrame from a dictionary including Series:
- >>> d = {'col1': [0, 1, 2, 3], 'col2': pd.Series([2, 3], index=[2, 3])}
+ >>> d = {"col1": [0, 1, 2, 3], "col2": pd.Series([2, 3], index=[2, 3])}
>>> pd.DataFrame(data=d, index=[0, 1, 2, 3])
col1 col2
0 0 NaN
@@ -595,8 +590,9 @@ class DataFrame(NDFrame, OpsMixin):
Constructing DataFrame from numpy ndarray:
- >>> df2 = pd.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
- ... columns=['a', 'b', 'c'])
+ >>> df2 = pd.DataFrame(
+ ... np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]), columns=["a", "b", "c"]
+ ... )
>>> df2
a b c
0 1 2 3
@@ -605,10 +601,11 @@ class DataFrame(NDFrame, OpsMixin):
Constructing DataFrame from a numpy ndarray that has labeled columns:
- >>> data = np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)],
- ... dtype=[("a", "i4"), ("b", "i4"), ("c", "i4")])
- >>> df3 = pd.DataFrame(data, columns=['c', 'a'])
- ...
+ >>> data = np.array(
+ ... [(1, 2, 3), (4, 5, 6), (7, 8, 9)],
+ ... dtype=[("a", "i4"), ("b", "i4"), ("c", "i4")],
+ ... )
+ >>> df3 = pd.DataFrame(data, columns=["c", "a"])
>>> df3
c a
0 3 1
@@ -647,36 +644,47 @@ class DataFrame(NDFrame, OpsMixin):
_HANDLED_TYPES = (Series, Index, ExtensionArray, np.ndarray)
_accessors: set[str] = {"sparse"}
_hidden_attrs: frozenset[str] = NDFrame._hidden_attrs | frozenset([])
- _mgr: BlockManager | ArrayManager
+ _mgr: BlockManager
# similar to __array_priority__, positions DataFrame before Series, Index,
# and ExtensionArray. Should NOT be overridden by subclasses.
__pandas_priority__ = 4000
@property
- def _constructor(self) -> Callable[..., DataFrame]:
+ def _constructor(self) -> type[DataFrame]:
return DataFrame
- def _constructor_from_mgr(self, mgr, axes):
- if self._constructor is DataFrame:
- # we are pandas.DataFrame (or a subclass that doesn't override _constructor)
- return DataFrame._from_mgr(mgr, axes=axes)
- else:
- assert axes is mgr.axes
+ def _constructor_from_mgr(self, mgr, axes) -> DataFrame:
+ df = DataFrame._from_mgr(mgr, axes=axes)
+
+ if type(self) is DataFrame:
+ # This would also work `if self._constructor is DataFrame`, but
+ # this check is slightly faster, benefiting the most-common case.
+ return df
+
+ elif type(self).__name__ == "GeoDataFrame":
+ # Shim until geopandas can override their _constructor_from_mgr
+ # bc they have different behavior for Managers than for DataFrames
return self._constructor(mgr)
+ # We assume that the subclass __init__ knows how to handle a
+ # pd.DataFrame object.
+ return self._constructor(df)
+
_constructor_sliced: Callable[..., Series] = Series
- def _sliced_from_mgr(self, mgr, axes) -> Series:
- return Series._from_mgr(mgr, axes)
+ def _constructor_sliced_from_mgr(self, mgr, axes) -> Series:
+ ser = Series._from_mgr(mgr, axes)
+ ser._name = None # caller is responsible for setting real name
- def _constructor_sliced_from_mgr(self, mgr, axes):
- if self._constructor_sliced is Series:
- ser = self._sliced_from_mgr(mgr, axes)
- ser._name = None # caller is responsible for setting real name
+ if type(self) is DataFrame:
+ # This would also work `if self._constructor_sliced is Series`, but
+ # this check is slightly faster, benefiting the most-common case.
return ser
- assert axes is mgr.axes
- return self._constructor_sliced(mgr)
+
+ # We assume that the subclass __init__ knows how to handle a
+ # pd.Series object.
+ return self._constructor_sliced(ser)
# ----------------------------------------------------------------------
# Constructors
@@ -701,7 +709,7 @@ def __init__(
# to avoid the result sharing the same Manager
data = data.copy(deep=False)
- if isinstance(data, (BlockManager, ArrayManager)):
+ if isinstance(data, BlockManager):
if not allow_mgr:
# GH#52419
warnings.warn(
@@ -712,8 +720,7 @@ def __init__(
stacklevel=1, # bump to 2 once pyarrow 15.0 is released with fix
)
- if using_copy_on_write():
- data = data.copy(deep=False)
+ data = data.copy(deep=False)
# first check if a Manager is passed without any other arguments
# -> use fastpath (without checking Manager type)
if index is None and columns is None and dtype is None and not copy:
@@ -721,8 +728,6 @@ def __init__(
NDFrame.__init__(self, data)
return
- manager = _get_option("mode.data_manager", silent=True)
-
is_pandas_object = isinstance(data, (Series, Index, ExtensionArray))
data_dtype = getattr(data, "dtype", None)
original_dtype = dtype
@@ -737,17 +742,7 @@ def __init__(
if isinstance(data, dict):
# retain pre-GH#38939 default behavior
copy = True
- elif (
- manager == "array"
- and isinstance(data, (np.ndarray, ExtensionArray))
- and data.ndim == 2
- ):
- # INFO(ArrayManager) by default copy the 2D input array to get
- # contiguous 1D arrays
- copy = True
- elif using_copy_on_write() and not isinstance(
- data, (Index, DataFrame, Series)
- ):
+ elif not isinstance(data, (Index, DataFrame, Series)):
copy = True
else:
copy = False
@@ -758,14 +753,14 @@ def __init__(
dtype = dtype if dtype is not None else pandas_dtype(object)
data = []
- if isinstance(data, (BlockManager, ArrayManager)):
+ if isinstance(data, BlockManager):
mgr = self._init_mgr(
data, axes={"index": index, "columns": columns}, dtype=dtype, copy=copy
)
elif isinstance(data, dict):
# GH#38939 de facto copy defaults to False only in non-dict cases
- mgr = dict_to_mgr(data, index, columns, dtype=dtype, copy=copy, typ=manager)
+ mgr = dict_to_mgr(data, index, columns, dtype=dtype, copy=copy)
elif isinstance(data, ma.MaskedArray):
from numpy.ma import mrecords
@@ -785,7 +780,6 @@ def __init__(
columns,
dtype=dtype,
copy=copy,
- typ=manager,
)
elif isinstance(data, (np.ndarray, Series, Index, ExtensionArray)):
@@ -798,11 +792,9 @@ def __init__(
columns,
dtype,
copy,
- typ=manager,
)
elif getattr(data, "name", None) is not None:
# i.e. Series/Index with non-None name
- _copy = copy if using_copy_on_write() else True
mgr = dict_to_mgr(
# error: Item "ndarray" of "Union[ndarray, Series, Index]" has no
# attribute "name"
@@ -810,8 +802,7 @@ def __init__(
index,
columns,
dtype=dtype,
- typ=manager,
- copy=_copy,
+ copy=copy,
)
else:
mgr = ndarray_to_mgr(
@@ -820,7 +811,6 @@ def __init__(
columns,
dtype=dtype,
copy=copy,
- typ=manager,
)
# For data is list-like, or Iterable (will consume into list)
@@ -851,7 +841,6 @@ def __init__(
columns,
index,
dtype=dtype,
- typ=manager,
)
else:
mgr = ndarray_to_mgr(
@@ -860,7 +849,6 @@ def __init__(
columns,
dtype=dtype,
copy=copy,
- typ=manager,
)
else:
mgr = dict_to_mgr(
@@ -868,7 +856,6 @@ def __init__(
index,
columns if columns is not None else default_index(0),
dtype=dtype,
- typ=manager,
)
# For data is scalar
else:
@@ -889,7 +876,7 @@ def __init__(
construct_1d_arraylike_from_scalar(data, len(index), dtype)
for _ in range(len(columns))
]
- mgr = arrays_to_mgr(values, columns, index, dtype=None, typ=manager)
+ mgr = arrays_to_mgr(values, columns, index, dtype=None)
else:
arr2d = construct_2d_arraylike_from_scalar(
data,
@@ -905,12 +892,8 @@ def __init__(
columns,
dtype=arr2d.dtype,
copy=False,
- typ=manager,
)
- # ensure correct Manager type according to settings
- mgr = mgr_to_mgr(mgr, typ=manager)
-
NDFrame.__init__(self, mgr)
if original_dtype is None and is_pandas_object and data_dtype == np.object_:
@@ -954,12 +937,13 @@ def __dataframe__(
Examples
--------
- >>> df_not_necessarily_pandas = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
+ >>> df_not_necessarily_pandas = pd.DataFrame({"A": [1, 2], "B": [3, 4]})
>>> interchange_object = df_not_necessarily_pandas.__dataframe__()
>>> interchange_object.column_names()
Index(['A', 'B'], dtype='object')
- >>> df_pandas = (pd.api.interchange.from_dataframe
- ... (interchange_object.select_columns_by_name(['A'])))
+ >>> df_pandas = pd.api.interchange.from_dataframe(
+ ... interchange_object.select_columns_by_name(["A"])
+ ... )
>>> df_pandas
A
0 1
@@ -973,20 +957,32 @@ def __dataframe__(
return PandasDataFrameXchg(self, allow_copy=allow_copy)
- def __dataframe_consortium_standard__(
- self, *, api_version: str | None = None
- ) -> Any:
+ def __arrow_c_stream__(self, requested_schema=None):
"""
- Provide entry point to the Consortium DataFrame Standard API.
+ Export the pandas DataFrame as an Arrow C stream PyCapsule.
+
+ This relies on pyarrow to convert the pandas DataFrame to the Arrow
+ format (and follows the default behaviour of ``pyarrow.Table.from_pandas``
+ in its handling of the index, i.e. store the index as a column except
+ for RangeIndex).
+ This conversion is not necessarily zero-copy.
+
+ Parameters
+ ----------
+ requested_schema : PyCapsule, default None
+ The schema to which the dataframe should be casted, passed as a
+ PyCapsule containing a C ArrowSchema representation of the
+ requested schema.
- This is developed and maintained outside of pandas.
- Please report any issues to https://github.com/data-apis/dataframe-api-compat.
+ Returns
+ -------
+ PyCapsule
"""
- dataframe_api_compat = import_optional_dependency("dataframe_api_compat")
- convert_to_standard_compliant_dataframe = (
- dataframe_api_compat.pandas_standard.convert_to_standard_compliant_dataframe
- )
- return convert_to_standard_compliant_dataframe(self, api_version=api_version)
+ pa = import_optional_dependency("pyarrow", min_version="14.0.0")
+ if requested_schema is not None:
+ requested_schema = pa.Schema._import_from_c_capsule(requested_schema)
+ table = pa.Table.from_pandas(self, schema=requested_schema)
+ return table.__arrow_c_stream__()
# ----------------------------------------------------------------------
@@ -1000,7 +996,7 @@ def axes(self) -> list[Index]:
Examples
--------
- >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
+ >>> df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4]})
>>> df.axes
[RangeIndex(start=0, stop=2, step=1), Index(['col1', 'col2'],
dtype='object')]
@@ -1018,12 +1014,11 @@ def shape(self) -> tuple[int, int]:
Examples
--------
- >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
+ >>> df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4]})
>>> df.shape
(2, 2)
- >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4],
- ... 'col3': [5, 6]})
+ >>> df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4], "col3": [5, 6]})
>>> df.shape
(2, 3)
"""
@@ -1048,9 +1043,12 @@ def _is_homogeneous_type(self) -> bool:
Items with the same type but different sizes are considered
different types.
- >>> DataFrame({
- ... "A": np.array([1, 2], dtype=np.int32),
- ... "B": np.array([1, 2], dtype=np.int64)})._is_homogeneous_type
+ >>> DataFrame(
+ ... {
+ ... "A": np.array([1, 2], dtype=np.int32),
+ ... "B": np.array([1, 2], dtype=np.int64),
+ ... }
+ ... )._is_homogeneous_type
False
"""
# The "<" part of "<=" here is for empty DataFrame cases
@@ -1061,8 +1059,6 @@ def _can_fast_transpose(self) -> bool:
"""
Can we transpose this DataFrame without creating any new array objects.
"""
- if isinstance(self._mgr, ArrayManager):
- return False
blocks = self._mgr.blocks
if len(blocks) != 1:
return False
@@ -1078,13 +1074,6 @@ def _values(self) -> np.ndarray | DatetimeArray | TimedeltaArray | PeriodArray:
"""
mgr = self._mgr
- if isinstance(mgr, ArrayManager):
- if len(mgr.arrays) == 1 and not is_1d_only_ea_dtype(mgr.arrays[0].dtype):
- # error: Item "ExtensionArray" of "Union[ndarray, ExtensionArray]"
- # has no attribute "reshape"
- return mgr.arrays[0].reshape(-1, 1) # type: ignore[union-attr]
- return ensure_wrapped_if_datetimelike(self.values)
-
blocks = mgr.blocks
if len(blocks) != 1:
return ensure_wrapped_if_datetimelike(self.values)
@@ -1224,6 +1213,7 @@ def _repr_html_(self) -> str | None:
def to_string(
self,
buf: None = ...,
+ *,
columns: Axes | None = ...,
col_space: int | list[int] | dict[Hashable, int] | None = ...,
header: bool | SequenceNotStr[str] = ...,
@@ -1242,13 +1232,13 @@ def to_string(
min_rows: int | None = ...,
max_colwidth: int | None = ...,
encoding: str | None = ...,
- ) -> str:
- ...
+ ) -> str: ...
@overload
def to_string(
self,
buf: FilePath | WriteBuffer[str],
+ *,
columns: Axes | None = ...,
col_space: int | list[int] | dict[Hashable, int] | None = ...,
header: bool | SequenceNotStr[str] = ...,
@@ -1267,12 +1257,8 @@ def to_string(
min_rows: int | None = ...,
max_colwidth: int | None = ...,
encoding: str | None = ...,
- ) -> None:
- ...
+ ) -> None: ...
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "buf"], name="to_string"
- )
@Substitution(
header_type="bool or list of str",
header="Write out the column names. If a list of columns "
@@ -1287,6 +1273,7 @@ def to_string(
def to_string(
self,
buf: FilePath | WriteBuffer[str] | None = None,
+ *,
columns: Axes | None = None,
col_space: int | list[int] | dict[Hashable, int] | None = None,
header: bool | SequenceNotStr[str] = True,
@@ -1325,7 +1312,7 @@ def to_string(
Examples
--------
- >>> d = {'col1': [1, 2, 3], 'col2': [4, 5, 6]}
+ >>> d = {"col1": [1, 2, 3], "col2": [4, 5, 6]}
>>> df = pd.DataFrame(d)
>>> print(df.to_string())
col1 col2
@@ -1368,7 +1355,7 @@ def _get_values_for_csv(
decimal: str,
na_rep: str,
quoting, # int csv.QUOTE_FOO from stdlib
- ) -> Self:
+ ) -> DataFrame:
# helper used by to_csv
mgr = self._mgr.get_values_for_csv(
float_format=float_format,
@@ -1395,7 +1382,7 @@ def style(self) -> Styler:
Examples
--------
- >>> df = pd.DataFrame({'A': [1, 2, 3]})
+ >>> df = pd.DataFrame({"A": [1, 2, 3]})
>>> df.style # doctest: +SKIP
Please see
@@ -1405,9 +1392,7 @@ def style(self) -> Styler:
return Styler(self)
- _shared_docs[
- "items"
- ] = r"""
+ _shared_docs["items"] = r"""
Iterate over (column name, Series) pairs.
Iterates over the DataFrame columns, returning a tuple with
@@ -1457,12 +1442,8 @@ def style(self) -> Styler:
@Appender(_shared_docs["items"])
def items(self) -> Iterable[tuple[Hashable, Series]]:
- if self.columns.is_unique and hasattr(self, "_item_cache"):
- for k in self.columns:
- yield k, self._get_item_cache(k)
- else:
- for i, k in enumerate(self.columns):
- yield k, self._ixs(i, axis=1)
+ for i, k in enumerate(self.columns):
+ yield k, self._ixs(i, axis=1)
def iterrows(self) -> Iterable[tuple[Hashable, Series]]:
"""
@@ -1498,24 +1479,23 @@ def iterrows(self) -> Iterable[tuple[Hashable, Series]]:
Examples
--------
- >>> df = pd.DataFrame([[1, 1.5]], columns=['int', 'float'])
+ >>> df = pd.DataFrame([[1, 1.5]], columns=["int", "float"])
>>> row = next(df.iterrows())[1]
>>> row
int 1.0
float 1.5
Name: 0, dtype: float64
- >>> print(row['int'].dtype)
+ >>> print(row["int"].dtype)
float64
- >>> print(df['int'].dtype)
+ >>> print(df["int"].dtype)
int64
"""
columns = self.columns
klass = self._constructor_sliced
- using_cow = using_copy_on_write()
for k, v in zip(self.index, self.values):
s = klass(v, index=columns, name=k).__finalize__(self)
- if using_cow and self._mgr.is_single_block:
- s._mgr.add_references(self._mgr) # type: ignore[arg-type]
+ if self._mgr.is_single_block:
+ s._mgr.add_references(self._mgr)
yield k, s
def itertuples(
@@ -1552,15 +1532,15 @@ def itertuples(
Examples
--------
- >>> df = pd.DataFrame({'num_legs': [4, 2], 'num_wings': [0, 2]},
- ... index=['dog', 'hawk'])
+ >>> df = pd.DataFrame(
+ ... {"num_legs": [4, 2], "num_wings": [0, 2]}, index=["dog", "hawk"]
+ ... )
>>> df
num_legs num_wings
dog 4 0
hawk 2 2
>>> for row in df.itertuples():
... print(row)
- ...
Pandas(Index='dog', num_legs=4, num_wings=0)
Pandas(Index='hawk', num_legs=2, num_wings=2)
@@ -1569,16 +1549,14 @@ def itertuples(
>>> for row in df.itertuples(index=False):
... print(row)
- ...
Pandas(num_legs=4, num_wings=0)
Pandas(num_legs=2, num_wings=2)
With the `name` parameter set we set a custom name for the yielded
namedtuples:
- >>> for row in df.itertuples(name='Animal'):
+ >>> for row in df.itertuples(name="Animal"):
... print(row)
- ...
Animal(Index='dog', num_legs=4, num_wings=0)
Animal(Index='hawk', num_legs=2, num_wings=2)
"""
@@ -1609,12 +1587,10 @@ def __len__(self) -> int:
return len(self.index)
@overload
- def dot(self, other: Series) -> Series:
- ...
+ def dot(self, other: Series) -> Series: ...
@overload
- def dot(self, other: DataFrame | Index | ArrayLike) -> DataFrame:
- ...
+ def dot(self, other: DataFrame | Index | ArrayLike) -> DataFrame: ...
def dot(self, other: AnyArrayLike | DataFrame) -> DataFrame | Series:
"""
@@ -1698,8 +1674,8 @@ def dot(self, other: AnyArrayLike | DataFrame) -> DataFrame | Series:
if len(common) > len(self.columns) or len(common) > len(other.index):
raise ValueError("matrices are not aligned")
- left = self.reindex(columns=common, copy=False)
- right = other.reindex(index=common, copy=False)
+ left = self.reindex(columns=common)
+ right = other.reindex(index=common)
lvals = left.values
rvals = right._values
else:
@@ -1735,12 +1711,10 @@ def dot(self, other: AnyArrayLike | DataFrame) -> DataFrame | Series:
raise TypeError(f"unsupported type: {type(other)}")
@overload
- def __matmul__(self, other: Series) -> Series:
- ...
+ def __matmul__(self, other: Series) -> Series: ...
@overload
- def __matmul__(self, other: AnyArrayLike | DataFrame) -> DataFrame | Series:
- ...
+ def __matmul__(self, other: AnyArrayLike | DataFrame) -> DataFrame | Series: ...
def __matmul__(self, other: AnyArrayLike | DataFrame) -> DataFrame | Series:
"""
@@ -1813,7 +1787,7 @@ def from_dict(
--------
By default the keys of the dict become the DataFrame columns:
- >>> data = {'col_1': [3, 2, 1, 0], 'col_2': ['a', 'b', 'c', 'd']}
+ >>> data = {"col_1": [3, 2, 1, 0], "col_2": ["a", "b", "c", "d"]}
>>> pd.DataFrame.from_dict(data)
col_1 col_2
0 3 a
@@ -1824,8 +1798,8 @@ def from_dict(
Specify ``orient='index'`` to create the DataFrame using dictionary
keys as rows:
- >>> data = {'row_1': [3, 2, 1, 0], 'row_2': ['a', 'b', 'c', 'd']}
- >>> pd.DataFrame.from_dict(data, orient='index')
+ >>> data = {"row_1": [3, 2, 1, 0], "row_2": ["a", "b", "c", "d"]}
+ >>> pd.DataFrame.from_dict(data, orient="index")
0 1 2 3
row_1 3 2 1 0
row_2 a b c d
@@ -1833,8 +1807,7 @@ def from_dict(
When using the 'index' orientation, the column names can be
specified manually:
- >>> pd.DataFrame.from_dict(data, orient='index',
- ... columns=['A', 'B', 'C', 'D'])
+ >>> pd.DataFrame.from_dict(data, orient="index", columns=["A", "B", "C", "D"])
A B C D
row_1 3 2 1 0
row_2 a b c d
@@ -1842,19 +1815,21 @@ def from_dict(
Specify ``orient='tight'`` to create the DataFrame using a 'tight'
format:
- >>> data = {'index': [('a', 'b'), ('a', 'c')],
- ... 'columns': [('x', 1), ('y', 2)],
- ... 'data': [[1, 3], [2, 4]],
- ... 'index_names': ['n1', 'n2'],
- ... 'column_names': ['z1', 'z2']}
- >>> pd.DataFrame.from_dict(data, orient='tight')
+ >>> data = {
+ ... "index": [("a", "b"), ("a", "c")],
+ ... "columns": [("x", 1), ("y", 2)],
+ ... "data": [[1, 3], [2, 4]],
+ ... "index_names": ["n1", "n2"],
+ ... "column_names": ["z1", "z2"],
+ ... }
+ >>> pd.DataFrame.from_dict(data, orient="tight")
z1 x y
z2 1 2
n1 n2
a b 1 3
c 2 4
"""
- index = None
+ index: list | Index | None = None
orient = orient.lower() # type: ignore[assignment]
if orient == "index":
if len(data) > 0:
@@ -1880,7 +1855,7 @@ def from_dict(
else:
realdata = data["data"]
- def create_index(indexlist, namelist):
+ def create_index(indexlist, namelist) -> Index:
index: Index
if len(namelist) > 1:
index = MultiIndex.from_tuples(indexlist, names=namelist)
@@ -1923,6 +1898,7 @@ def to_numpy(
Returns
-------
numpy.ndarray
+ The NumPy array representing the values in the DataFrame.
See Also
--------
@@ -1945,7 +1921,7 @@ def to_numpy(
For a mix of numeric and non-numeric types, the output array will
have object dtype.
- >>> df['C'] = pd.date_range('2000', periods=2)
+ >>> df["C"] = pd.date_range("2000", periods=2)
>>> df.to_numpy()
array([[1, 3.0, Timestamp('2000-01-01 00:00:00')],
[2, 4.5, Timestamp('2000-01-02 00:00:00')]], dtype=object)
@@ -1954,32 +1930,10 @@ def to_numpy(
dtype = np.dtype(dtype)
result = self._mgr.as_array(dtype=dtype, copy=copy, na_value=na_value)
if result.dtype is not dtype:
- result = np.array(result, dtype=dtype, copy=False)
+ result = np.asarray(result, dtype=dtype)
return result
- def _create_data_for_split_and_tight_to_dict(
- self, are_all_object_dtype_cols: bool, object_dtype_indices: list[int]
- ) -> list:
- """
- Simple helper method to create data for to ``to_dict(orient="split")`` and
- ``to_dict(orient="tight")`` to create the main output data
- """
- if are_all_object_dtype_cols:
- data = [
- list(map(maybe_box_native, t))
- for t in self.itertuples(index=False, name=None)
- ]
- else:
- data = [list(t) for t in self.itertuples(index=False, name=None)]
- if object_dtype_indices:
- # If we have object_dtype_cols, apply maybe_box_naive after list
- # comprehension for perf
- for row in data:
- for i in object_dtype_indices:
- row[i] = maybe_box_native(row[i])
- return data
-
@overload
def to_dict(
self,
@@ -1987,8 +1941,7 @@ def to_dict(
*,
into: type[MutableMappingT] | MutableMappingT,
index: bool = ...,
- ) -> MutableMappingT:
- ...
+ ) -> MutableMappingT: ...
@overload
def to_dict(
@@ -1997,8 +1950,7 @@ def to_dict(
*,
into: type[MutableMappingT] | MutableMappingT,
index: bool = ...,
- ) -> list[MutableMappingT]:
- ...
+ ) -> list[MutableMappingT]: ...
@overload
def to_dict(
@@ -2007,8 +1959,7 @@ def to_dict(
*,
into: type[dict] = ...,
index: bool = ...,
- ) -> dict:
- ...
+ ) -> dict: ...
@overload
def to_dict(
@@ -2017,21 +1968,17 @@ def to_dict(
*,
into: type[dict] = ...,
index: bool = ...,
- ) -> list[dict]:
- ...
+ ) -> list[dict]: ...
# error: Incompatible default for argument "into" (default has type "type
# [dict[Any, Any]]", argument has type "type[MutableMappingT] | MutableMappingT")
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "orient"], name="to_dict"
- )
def to_dict(
self,
orient: Literal[
"dict", "list", "series", "split", "tight", "records", "index"
] = "dict",
- into: type[MutableMappingT]
- | MutableMappingT = dict, # type: ignore[assignment]
+ *,
+ into: type[MutableMappingT] | MutableMappingT = dict, # type: ignore[assignment]
index: bool = True,
) -> MutableMappingT | list[MutableMappingT]:
"""
@@ -2069,7 +2016,9 @@ def to_dict(
index : bool, default True
Whether to include the index item (and index_names item if `orient`
is 'tight') in the returned dictionary. Can only be ``False``
- when `orient` is 'split' or 'tight'.
+ when `orient` is 'split' or 'tight'. Note that when `orient` is
+ 'records', this parameter does not take effect (index item always
+ not included).
.. versionadded:: 2.0.0
@@ -2087,9 +2036,9 @@ def to_dict(
Examples
--------
- >>> df = pd.DataFrame({'col1': [1, 2],
- ... 'col2': [0.5, 0.75]},
- ... index=['row1', 'row2'])
+ >>> df = pd.DataFrame(
+ ... {"col1": [1, 2], "col2": [0.5, 0.75]}, index=["row1", "row2"]
+ ... )
>>> df
col1 col2
row1 1 0.50
@@ -2099,7 +2048,7 @@ def to_dict(
You can specify the return orientation.
- >>> df.to_dict('series')
+ >>> df.to_dict("series")
{'col1': row1 1
row2 2
Name: col1, dtype: int64,
@@ -2107,17 +2056,17 @@ def to_dict(
row2 0.75
Name: col2, dtype: float64}
- >>> df.to_dict('split')
+ >>> df.to_dict("split")
{'index': ['row1', 'row2'], 'columns': ['col1', 'col2'],
'data': [[1, 0.5], [2, 0.75]]}
- >>> df.to_dict('records')
+ >>> df.to_dict("records")
[{'col1': 1, 'col2': 0.5}, {'col1': 2, 'col2': 0.75}]
- >>> df.to_dict('index')
+ >>> df.to_dict("index")
{'row1': {'col1': 1, 'col2': 0.5}, 'row2': {'col1': 2, 'col2': 0.75}}
- >>> df.to_dict('tight')
+ >>> df.to_dict("tight")
{'index': ['row1', 'row2'], 'columns': ['col1', 'col2'],
'data': [[1, 0.5], [2, 0.75]], 'index_names': [None], 'column_names': [None]}
@@ -2131,7 +2080,7 @@ def to_dict(
If you want a `defaultdict`, you need to initialize it:
>>> dd = defaultdict(list)
- >>> df.to_dict('records', into=dd)
+ >>> df.to_dict("records", into=dd)
[defaultdict(, {'col1': 1, 'col2': 0.5}),
defaultdict(, {'col1': 2, 'col2': 0.75})]
"""
@@ -2139,144 +2088,6 @@ def to_dict(
return to_dict(self, orient, into=into, index=index)
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "destination_table"], name="to_gbq"
- )
- def to_gbq(
- self,
- destination_table: str,
- project_id: str | None = None,
- chunksize: int | None = None,
- reauth: bool = False,
- if_exists: ToGbqIfexist = "fail",
- auth_local_webserver: bool = True,
- table_schema: list[dict[str, str]] | None = None,
- location: str | None = None,
- progress_bar: bool = True,
- credentials=None,
- ) -> None:
- """
- Write a DataFrame to a Google BigQuery table.
-
- .. deprecated:: 2.2.0
-
- Please use ``pandas_gbq.to_gbq`` instead.
-
- This function requires the `pandas-gbq package
- `__.
-
- See the `How to authenticate with Google BigQuery
- `__
- guide for authentication instructions.
-
- Parameters
- ----------
- destination_table : str
- Name of table to be written, in the form ``dataset.tablename``.
- project_id : str, optional
- Google BigQuery Account project ID. Optional when available from
- the environment.
- chunksize : int, optional
- Number of rows to be inserted in each chunk from the dataframe.
- Set to ``None`` to load the whole dataframe at once.
- reauth : bool, default False
- Force Google BigQuery to re-authenticate the user. This is useful
- if multiple accounts are used.
- if_exists : str, default 'fail'
- Behavior when the destination table exists. Value can be one of:
-
- ``'fail'``
- If table exists raise pandas_gbq.gbq.TableCreationError.
- ``'replace'``
- If table exists, drop it, recreate it, and insert data.
- ``'append'``
- If table exists, insert data. Create if does not exist.
- auth_local_webserver : bool, default True
- Use the `local webserver flow`_ instead of the `console flow`_
- when getting user credentials.
-
- .. _local webserver flow:
- https://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_local_server
- .. _console flow:
- https://google-auth-oauthlib.readthedocs.io/en/latest/reference/google_auth_oauthlib.flow.html#google_auth_oauthlib.flow.InstalledAppFlow.run_console
-
- *New in version 0.2.0 of pandas-gbq*.
-
- .. versionchanged:: 1.5.0
- Default value is changed to ``True``. Google has deprecated the
- ``auth_local_webserver = False`` `"out of band" (copy-paste)
- flow
- `_.
- table_schema : list of dicts, optional
- List of BigQuery table fields to which according DataFrame
- columns conform to, e.g. ``[{'name': 'col1', 'type':
- 'STRING'},...]``. If schema is not provided, it will be
- generated according to dtypes of DataFrame columns. See
- BigQuery API documentation on available names of a field.
-
- *New in version 0.3.1 of pandas-gbq*.
- location : str, optional
- Location where the load job should run. See the `BigQuery locations
- documentation
- `__ for a
- list of available locations. The location must match that of the
- target dataset.
-
- *New in version 0.5.0 of pandas-gbq*.
- progress_bar : bool, default True
- Use the library `tqdm` to show the progress bar for the upload,
- chunk by chunk.
-
- *New in version 0.5.0 of pandas-gbq*.
- credentials : google.auth.credentials.Credentials, optional
- Credentials for accessing Google APIs. Use this parameter to
- override default credentials, such as to use Compute Engine
- :class:`google.auth.compute_engine.Credentials` or Service
- Account :class:`google.oauth2.service_account.Credentials`
- directly.
-
- *New in version 0.8.0 of pandas-gbq*.
-
- See Also
- --------
- pandas_gbq.to_gbq : This function in the pandas-gbq library.
- read_gbq : Read a DataFrame from Google BigQuery.
-
- Examples
- --------
- Example taken from `Google BigQuery documentation
- `_
-
- >>> project_id = "my-project"
- >>> table_id = 'my_dataset.my_table'
- >>> df = pd.DataFrame({
- ... "my_string": ["a", "b", "c"],
- ... "my_int64": [1, 2, 3],
- ... "my_float64": [4.0, 5.0, 6.0],
- ... "my_bool1": [True, False, True],
- ... "my_bool2": [False, True, False],
- ... "my_dates": pd.date_range("now", periods=3),
- ... }
- ... )
-
- >>> df.to_gbq(table_id, project_id=project_id) # doctest: +SKIP
- """
- from pandas.io import gbq
-
- gbq.to_gbq(
- self,
- destination_table,
- project_id=project_id,
- chunksize=chunksize,
- reauth=reauth,
- if_exists=if_exists,
- auth_local_webserver=auth_local_webserver,
- table_schema=table_schema,
- location=location,
- progress_bar=progress_bar,
- credentials=credentials,
- )
-
@classmethod
def from_records(
cls,
@@ -2295,11 +2106,8 @@ def from_records(
Parameters
----------
- data : structured ndarray, sequence of tuples or dicts, or DataFrame
+ data : structured ndarray, sequence of tuples or dicts
Structured input data.
-
- .. deprecated:: 2.1.0
- Passing a DataFrame is deprecated.
index : str, list of fields, array-like
Field of array to use as the index, alternately a specific set of
input labels to use.
@@ -2330,8 +2138,10 @@ def from_records(
--------
Data can be provided as a structured ndarray:
- >>> data = np.array([(3, 'a'), (2, 'b'), (1, 'c'), (0, 'd')],
- ... dtype=[('col_1', 'i4'), ('col_2', 'U1')])
+ >>> data = np.array(
+ ... [(3, "a"), (2, "b"), (1, "c"), (0, "d")],
+ ... dtype=[("col_1", "i4"), ("col_2", "U1")],
+ ... )
>>> pd.DataFrame.from_records(data)
col_1 col_2
0 3 a
@@ -2341,10 +2151,12 @@ def from_records(
Data can be provided as a list of dicts:
- >>> data = [{'col_1': 3, 'col_2': 'a'},
- ... {'col_1': 2, 'col_2': 'b'},
- ... {'col_1': 1, 'col_2': 'c'},
- ... {'col_1': 0, 'col_2': 'd'}]
+ >>> data = [
+ ... {"col_1": 3, "col_2": "a"},
+ ... {"col_1": 2, "col_2": "b"},
+ ... {"col_1": 1, "col_2": "c"},
+ ... {"col_1": 0, "col_2": "d"},
+ ... ]
>>> pd.DataFrame.from_records(data)
col_1 col_2
0 3 a
@@ -2354,8 +2166,8 @@ def from_records(
Data can be provided as a list of tuples with corresponding columns:
- >>> data = [(3, 'a'), (2, 'b'), (1, 'c'), (0, 'd')]
- >>> pd.DataFrame.from_records(data, columns=['col_1', 'col_2'])
+ >>> data = [(3, "a"), (2, "b"), (1, "c"), (0, "d")]
+ >>> pd.DataFrame.from_records(data, columns=["col_1", "col_2"])
col_1 col_2
0 3 a
1 2 b
@@ -2363,21 +2175,10 @@ def from_records(
3 0 d
"""
if isinstance(data, DataFrame):
- warnings.warn(
- "Passing a DataFrame to DataFrame.from_records is deprecated. Use "
+ raise TypeError(
+ "Passing a DataFrame to DataFrame.from_records is not supported. Use "
"set_index and/or drop to modify the DataFrame instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
)
- if columns is not None:
- if is_scalar(columns):
- columns = [columns]
- data = data[columns]
- if index is not None:
- data = data.set_index(index)
- if exclude is not None:
- data = data.drop(columns=exclude)
- return data.copy(deep=False)
result_index = None
@@ -2500,9 +2301,7 @@ def maybe_reorder(
columns = columns.drop(exclude)
- manager = _get_option("mode.data_manager", silent=True)
- mgr = arrays_to_mgr(arrays, columns, result_index, typ=manager)
-
+ mgr = arrays_to_mgr(arrays, columns, result_index)
return cls._from_mgr(mgr, axes=mgr.axes)
def to_records(
@@ -2546,8 +2345,7 @@ def to_records(
Examples
--------
- >>> df = pd.DataFrame({'A': [1, 2], 'B': [0.5, 0.75]},
- ... index=['a', 'b'])
+ >>> df = pd.DataFrame({"A": [1, 2], "B": [0.5, 0.75]}, index=["a", "b"])
>>> df
A B
a 1 0.50
@@ -2701,7 +2499,6 @@ def _from_arrays(
if dtype is not None:
dtype = pandas_dtype(dtype)
- manager = _get_option("mode.data_manager", silent=True)
columns = ensure_index(columns)
if len(columns) != len(arrays):
raise ValueError("len(columns) must match len(arrays)")
@@ -2711,7 +2508,6 @@ def _from_arrays(
index,
dtype=dtype,
verify_integrity=verify_integrity,
- typ=manager,
)
return cls._from_mgr(mgr, axes=mgr.axes)
@@ -2820,10 +2616,10 @@ def to_stata(
Examples
--------
- >>> df = pd.DataFrame({{'animal': ['falcon', 'parrot', 'falcon',
- ... 'parrot'],
- ... 'speed': [350, 18, 361, 15]}})
- >>> df.to_stata('animals.dta') # doctest: +SKIP
+ >>> df = pd.DataFrame(
+ ... [["falcon", 350], ["parrot", 18]], columns=["animal", "parrot"]
+ ... )
+ >>> df.to_stata("animals.dta") # doctest: +SKIP
"""
if version not in (114, 117, 118, 119, None):
raise ValueError("Only formats 114, 117, 118 and 119 are supported.")
@@ -2899,9 +2695,39 @@ def to_feather(self, path: FilePath | WriteBuffer[bytes], **kwargs) -> None:
to_feather(self, path, **kwargs)
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "buf"], name="to_markdown"
- )
+ @overload
+ def to_markdown(
+ self,
+ buf: None = ...,
+ *,
+ mode: str = ...,
+ index: bool = ...,
+ storage_options: StorageOptions | None = ...,
+ **kwargs,
+ ) -> str: ...
+
+ @overload
+ def to_markdown(
+ self,
+ buf: FilePath | WriteBuffer[str],
+ *,
+ mode: str = ...,
+ index: bool = ...,
+ storage_options: StorageOptions | None = ...,
+ **kwargs,
+ ) -> None: ...
+
+ @overload
+ def to_markdown(
+ self,
+ buf: FilePath | WriteBuffer[str] | None,
+ *,
+ mode: str = ...,
+ index: bool = ...,
+ storage_options: StorageOptions | None = ...,
+ **kwargs,
+ ) -> str | None: ...
+
@doc(
Series.to_markdown,
klass=_shared_doc_kwargs["klass"],
@@ -2931,6 +2757,7 @@ def to_feather(self, path: FilePath | WriteBuffer[bytes], **kwargs) -> None:
def to_markdown(
self,
buf: FilePath | WriteBuffer[str] | None = None,
+ *,
mode: str = "wt",
index: bool = True,
storage_options: StorageOptions | None = None,
@@ -2955,35 +2782,33 @@ def to_markdown(
def to_parquet(
self,
path: None = ...,
+ *,
engine: Literal["auto", "pyarrow", "fastparquet"] = ...,
compression: str | None = ...,
index: bool | None = ...,
partition_cols: list[str] | None = ...,
storage_options: StorageOptions = ...,
**kwargs,
- ) -> bytes:
- ...
+ ) -> bytes: ...
@overload
def to_parquet(
self,
path: FilePath | WriteBuffer[bytes],
+ *,
engine: Literal["auto", "pyarrow", "fastparquet"] = ...,
compression: str | None = ...,
index: bool | None = ...,
partition_cols: list[str] | None = ...,
storage_options: StorageOptions = ...,
**kwargs,
- ) -> None:
- ...
+ ) -> None: ...
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "path"], name="to_parquet"
- )
@doc(storage_options=_shared_docs["storage_options"])
def to_parquet(
self,
path: FilePath | WriteBuffer[bytes] | None = None,
+ *,
engine: Literal["auto", "pyarrow", "fastparquet"] = "auto",
compression: str | None = "snappy",
index: bool | None = None,
@@ -3052,10 +2877,9 @@ def to_parquet(
Examples
--------
- >>> df = pd.DataFrame(data={{'col1': [1, 2], 'col2': [3, 4]}})
- >>> df.to_parquet('df.parquet.gzip',
- ... compression='gzip') # doctest: +SKIP
- >>> pd.read_parquet('df.parquet.gzip') # doctest: +SKIP
+ >>> df = pd.DataFrame(data={{"col1": [1, 2], "col2": [3, 4]}})
+ >>> df.to_parquet("df.parquet.gzip", compression="gzip") # doctest: +SKIP
+ >>> pd.read_parquet("df.parquet.gzip") # doctest: +SKIP
col1 col2
0 1 3
1 2 4
@@ -3083,6 +2907,36 @@ def to_parquet(
**kwargs,
)
+ @overload
+ def to_orc(
+ self,
+ path: None = ...,
+ *,
+ engine: Literal["pyarrow"] = ...,
+ index: bool | None = ...,
+ engine_kwargs: dict[str, Any] | None = ...,
+ ) -> bytes: ...
+
+ @overload
+ def to_orc(
+ self,
+ path: FilePath | WriteBuffer[bytes],
+ *,
+ engine: Literal["pyarrow"] = ...,
+ index: bool | None = ...,
+ engine_kwargs: dict[str, Any] | None = ...,
+ ) -> None: ...
+
+ @overload
+ def to_orc(
+ self,
+ path: FilePath | WriteBuffer[bytes] | None,
+ *,
+ engine: Literal["pyarrow"] = ...,
+ index: bool | None = ...,
+ engine_kwargs: dict[str, Any] | None = ...,
+ ) -> bytes | None: ...
+
def to_orc(
self,
path: FilePath | WriteBuffer[bytes] | None = None,
@@ -3092,7 +2946,7 @@ def to_orc(
engine_kwargs: dict[str, Any] | None = None,
) -> bytes | None:
"""
- Write a DataFrame to the ORC format.
+ Write a DataFrame to the Optimized Row Columnar (ORC) format.
.. versionadded:: 1.5.0
@@ -3119,7 +2973,8 @@ def to_orc(
Returns
-------
- bytes if no path argument is provided else None
+ bytes if no ``path`` argument is provided else None
+ Bytes object with DataFrame data if ``path`` is not specified else None.
Raises
------
@@ -3139,6 +2994,8 @@ def to_orc(
Notes
-----
+ * Find more information on ORC
+ `here `__.
* Before using this function you should read the :ref:`user guide about
ORC ` and :ref:`install optional dependencies `.
* This function requires `pyarrow
...
...
@@ -3370,8 +3225,7 @@ def to_xml(
stylesheet: FilePath | ReadBuffer[str] | ReadBuffer[bytes] | None = ...,
compression: CompressionOptions = ...,
storage_options: StorageOptions | None = ...,
- ) -> str:
- ...
+ ) -> str: ...
@overload
def to_xml(
@@ -3393,12 +3247,8 @@ def to_xml(
stylesheet: FilePath | ReadBuffer[str] | ReadBuffer[bytes] | None = ...,
compression: CompressionOptions = ...,
storage_options: StorageOptions | None = ...,
- ) -> None:
- ...
+ ) -> None: ...
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "path_or_buffer"], name="to_xml"
- )
@doc(
storage_options=_shared_docs["storage_options"],
compression_options=_shared_docs["compression_options"] % "path_or_buffer",
@@ -3406,6 +3256,7 @@ def to_xml(
def to_xml(
self,
path_or_buffer: FilePath | WriteBuffer[bytes] | WriteBuffer[str] | None = None,
+ *,
index: bool = True,
root_name: str | None = "data",
row_name: str | None = "row",
@@ -3498,9 +3349,10 @@ def to_xml(
Examples
--------
- >>> df = pd.DataFrame({{'shape': ['square', 'circle', 'triangle'],
- ... 'degrees': [360, 360, 180],
- ... 'sides': [4, np.nan, 3]}})
+ >>> df = pd.DataFrame(
+ ... [["square", 360, 4], ["circle", 360, np.nan], ["triangle", 180, 3]],
+ ... columns=["shape", "degrees", "sides"],
+ ... )
>>> df.to_xml() # doctest: +SKIP
@@ -3525,9 +3377,9 @@ def to_xml(
- >>> df.to_xml(attr_cols=[
- ... 'index', 'shape', 'degrees', 'sides'
- ... ]) # doctest: +SKIP
+ >>> df.to_xml(
+ ... attr_cols=["index", "shape", "degrees", "sides"]
+ ... ) # doctest: +SKIP
|
@@ -3535,8 +3387,9 @@ def to_xml(
|
- >>> df.to_xml(namespaces={{"doc": "https://example.com"}},
- ... prefix="doc") # doctest: +SKIP
+ >>> df.to_xml(
+ ... namespaces={{"doc": "https://example.com"}}, prefix="doc"
+ ... ) # doctest: +SKIP
@@ -3668,9 +3521,8 @@ def memory_usage(self, index: bool = True, deep: bool = False) -> Series:
Examples
--------
- >>> dtypes = ['int64', 'float64', 'complex128', 'object', 'bool']
- >>> data = dict([(t, np.ones(shape=5000, dtype=int).astype(t))
- ... for t in dtypes])
+ >>> dtypes = ["int64", "float64", "complex128", "object", "bool"]
+ >>> data = dict([(t, np.ones(shape=5000, dtype=int).astype(t)) for t in dtypes])
>>> df = pd.DataFrame(data)
>>> df.head()
int64 float64 complex128 object bool
@@ -3711,7 +3563,7 @@ def memory_usage(self, index: bool = True, deep: bool = False) -> Series:
Use a Categorical for efficient storage of an object-dtype column with
many repeated values.
- >>> df['object'].astype('category').memory_usage(deep=True)
+ >>> df["object"].astype("category").memory_usage(deep=True)
5136
"""
result = self._constructor_sliced(
@@ -3726,7 +3578,11 @@ def memory_usage(self, index: bool = True, deep: bool = False) -> Series:
result = index_memory_usage._append(result)
return result
- def transpose(self, *args, copy: bool = False) -> DataFrame:
+ def transpose(
+ self,
+ *args,
+ copy: bool | lib.NoDefault = lib.no_default,
+ ) -> DataFrame:
"""
Transpose index and columns.
@@ -3757,6 +3613,8 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+ .. deprecated:: 3.0.0
+
Returns
-------
DataFrame
@@ -3776,7 +3634,7 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
--------
**Square DataFrame with homogeneous dtype**
- >>> d1 = {'col1': [1, 2], 'col2': [3, 4]}
+ >>> d1 = {"col1": [1, 2], "col2": [3, 4]}
>>> df1 = pd.DataFrame(data=d1)
>>> df1
col1 col2
@@ -3803,10 +3661,12 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
**Non-square DataFrame with mixed dtypes**
- >>> d2 = {'name': ['Alice', 'Bob'],
- ... 'score': [9.5, 8],
- ... 'employed': [False, True],
- ... 'kids': [0, 0]}
+ >>> d2 = {
+ ... "name": ["Alice", "Bob"],
+ ... "score": [9.5, 8],
+ ... "employed": [False, True],
+ ... "kids": [0, 0],
+ ... }
>>> df2 = pd.DataFrame(data=d2)
>>> df2
name score employed kids
@@ -3835,6 +3695,7 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
1 object
dtype: object
"""
+ self._check_copy_deprecation(copy)
nv.validate_transpose(args, {})
# construct the args
@@ -3843,8 +3704,6 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
if self._can_fast_transpose:
# Note: tests pass without this, but this improves perf quite a bit.
new_vals = self._values.T
- if copy and not using_copy_on_write():
- new_vals = new_vals.copy()
result = self._constructor(
new_vals,
@@ -3853,8 +3712,8 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
copy=False,
dtype=new_vals.dtype,
)
- if using_copy_on_write() and len(self) > 0:
- result._mgr.add_references(self._mgr) # type: ignore[arg-type]
+ if len(self) > 0:
+ result._mgr.add_references(self._mgr)
elif (
self._is_homogeneous_type
@@ -3897,8 +3756,6 @@ def transpose(self, *args, copy: bool = False) -> DataFrame:
else:
new_arr = self.values.T
- if copy and not using_copy_on_write():
- new_arr = new_arr.copy()
result = self._constructor(
new_arr,
index=self.columns,
@@ -3926,7 +3783,7 @@ def T(self) -> DataFrame:
Examples
--------
- >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
+ >>> df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4]})
>>> df
col1 col2
0 1 3
@@ -3957,24 +3814,14 @@ def _ixs(self, i: int, axis: AxisInt = 0) -> Series:
if axis == 0:
new_mgr = self._mgr.fast_xs(i)
- # if we are a copy, mark as such
- copy = isinstance(new_mgr.array, np.ndarray) and new_mgr.array.base is None
result = self._constructor_sliced_from_mgr(new_mgr, axes=new_mgr.axes)
result._name = self.index[i]
- result = result.__finalize__(self)
- result._set_is_copy(self, copy=copy)
- return result
+ return result.__finalize__(self)
# icol
else:
- label = self.columns[i]
-
col_mgr = self._mgr.iget(i)
- result = self._box_col_values(col_mgr, i)
-
- # this is a cached value, mark it so
- result._set_as_cached(label, self)
- return result
+ return self._box_col_values(col_mgr, i)
def _get_column_array(self, i: int) -> ArrayLike:
"""
@@ -3994,30 +3841,8 @@ def _iter_column_arrays(self) -> Iterator[ArrayLike]:
Warning! The returned array is a view but doesn't handle Copy-on-Write,
so this should be used with caution (for read-only purposes).
"""
- if isinstance(self._mgr, ArrayManager):
- yield from self._mgr.arrays
- else:
- for i in range(len(self.columns)):
- yield self._get_column_array(i)
-
- def _getitem_nocopy(self, key: list):
- """
- Behaves like __getitem__, but returns a view in cases where __getitem__
- would make a copy.
- """
- # TODO(CoW): can be removed if/when we are always Copy-on-Write
- indexer = self.columns._get_indexer_strict(key, "columns")[1]
- new_axis = self.columns[indexer]
-
- new_mgr = self._mgr.reindex_indexer(
- new_axis,
- indexer,
- axis=0,
- allow_dups=True,
- copy=False,
- only_slice=True,
- )
- return self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
+ for i in range(len(self.columns)):
+ yield self._get_column_array(i)
def __getitem__(self, key):
check_dict_or_set_indexers(key)
@@ -4035,7 +3860,7 @@ def __getitem__(self, key):
and key in self.columns
or key in self.columns.drop_duplicates(keep=False)
):
- return self._get_item_cache(key)
+ return self._get_item(key)
elif is_mi and self.columns.is_unique and key in self.columns:
return self._getitem_multilevel(key)
@@ -4074,7 +3899,7 @@ def __getitem__(self, key):
if isinstance(indexer, slice):
return self._slice(indexer, axis=1)
- data = self._take_with_is_copy(indexer, axis=1)
+ data = self.take(indexer, axis=1)
if is_single_key:
# What does looking for a single key in a non-unique index return?
@@ -4083,7 +3908,7 @@ def __getitem__(self, key):
# - we have a MultiIndex on columns (test on self.columns, #21309)
if data.shape[1] == 1 and not isinstance(self.columns, MultiIndex):
# GH#26490 using data[key] can cause RecursionError
- return data._get_item_cache(key)
+ return data._get_item(key)
return data
@@ -4109,10 +3934,10 @@ def _getitem_bool_array(self, key):
key = check_bool_indexer(self.index, key)
if key.all():
- return self.copy(deep=None)
+ return self.copy(deep=False)
indexer = key.nonzero()[0]
- return self._take_with_is_copy(indexer, axis=0)
+ return self.take(indexer, axis=0)
def _getitem_multilevel(self, key):
# self.columns is a MultiIndex
@@ -4142,7 +3967,6 @@ def _getitem_multilevel(self, key):
result, index=self.index, name=key
)
- result._set_is_copy(self)
return result
else:
# loc is neither a slice nor ndarray, so must be an int
@@ -4171,7 +3995,7 @@ def _get_value(self, index, col, takeable: bool = False) -> Scalar:
series = self._ixs(col, axis=1)
return series._values[index]
- series = self._get_item_cache(col)
+ series = self._get_item(col)
engine = self.index._engine
if not isinstance(self.index, MultiIndex):
@@ -4199,6 +4023,11 @@ def isetitem(self, loc, value) -> None:
value : scalar or arraylike
Value(s) for the column.
+ See Also
+ --------
+ DataFrame.iloc : Purely integer-location based indexing for selection by
+ position.
+
Notes
-----
``frame.isetitem(loc, value)`` is an in-place method as it will
@@ -4209,6 +4038,15 @@ def isetitem(self, loc, value) -> None:
In cases where ``frame.columns`` is unique, this is equivalent to
``frame[frame.columns[i]] = value``.
+
+ Examples
+ --------
+ >>> df = pd.DataFrame({"A": [1, 2], "B": [3, 4]})
+ >>> df.isetitem(1, [5, 6])
+ >>> df
+ A B
+ 0 1 5
+ 1 2 6
"""
if isinstance(value, DataFrame):
if is_integer(loc):
@@ -4229,22 +4067,11 @@ def isetitem(self, loc, value) -> None:
self._iset_item_mgr(loc, arraylike, inplace=False, refs=refs)
def __setitem__(self, key, value) -> None:
- if not PYPY and using_copy_on_write():
+ if not PYPY:
if sys.getrefcount(self) <= 3:
warnings.warn(
_chained_assignment_msg, ChainedAssignmentError, stacklevel=2
)
- elif not PYPY and not using_copy_on_write():
- if sys.getrefcount(self) <= 3 and (
- warn_copy_on_write()
- or (
- not warn_copy_on_write()
- and any(b.refs.has_reference() for b in self._mgr.blocks) # type: ignore[union-attr]
- )
- ):
- warnings.warn(
- _chained_assignment_warning_msg, FutureWarning, stacklevel=2
- )
key = com.apply_if_callable(key, self)
@@ -4274,10 +4101,9 @@ def _setitem_slice(self, key: slice, value) -> None:
# NB: we can't just use self.loc[key] = value because that
# operates on labels and we need to operate positional for
# backwards-compat, xref GH#31469
- self._check_setitem_copy()
self.iloc[key] = value
- def _setitem_array(self, key, value):
+ def _setitem_array(self, key, value) -> None:
# also raises Exception if object array with NA values
if com.is_bool_indexer(key):
# bool indexer is indexing along rows
@@ -4287,7 +4113,6 @@ def _setitem_array(self, key, value):
)
key = check_bool_indexer(self.index, key)
indexer = key.nonzero()[0]
- self._check_setitem_copy()
if isinstance(value, DataFrame):
# GH#39931 reindex since iloc does not align
value = value.reindex(self.index.take(indexer))
@@ -4312,7 +4137,7 @@ def _setitem_array(self, key, value):
elif np.ndim(value) > 1:
# list of lists
value = DataFrame(value).values
- return self._setitem_array(key, value)
+ self._setitem_array(key, value)
else:
self._iset_not_inplace(key, value)
@@ -4374,7 +4199,6 @@ def _setitem_frame(self, key, value) -> None:
"Must pass DataFrame or 2-d ndarray with boolean values only"
)
- self._check_setitem_copy()
self._where(-key, value, inplace=True)
def _set_item_frame_value(self, key, value: DataFrame) -> None:
@@ -4436,7 +4260,6 @@ def _iset_item_mgr(
) -> None:
# when called from _set_item_mgr loc can be anything returned from get_loc
self._mgr.iset(loc, value, inplace=inplace, refs=refs)
- self._clear_item_cache()
def _set_item_mgr(
self, key, value: ArrayLike, refs: BlockValuesRefs | None = None
@@ -4449,27 +4272,10 @@ def _set_item_mgr(
else:
self._iset_item_mgr(loc, value, refs=refs)
- # check if we are modifying a copy
- # try to set first as we want an invalid
- # value exception to occur first
- if len(self):
- self._check_setitem_copy()
-
def _iset_item(self, loc: int, value: Series, inplace: bool = True) -> None:
# We are only called from _replace_columnwise which guarantees that
# no reindex is necessary
- if using_copy_on_write():
- self._iset_item_mgr(
- loc, value._values, inplace=inplace, refs=value._references
- )
- else:
- self._iset_item_mgr(loc, value._values.copy(), inplace=True)
-
- # check if we are modifying a copy
- # try to set first as we want an invalid
- # value exception to occur first
- if len(self):
- self._check_setitem_copy()
+ self._iset_item_mgr(loc, value._values, inplace=inplace, refs=value._references)
def _set_item(self, key, value) -> None:
"""
@@ -4521,7 +4327,6 @@ def _set_value(
icol = self.columns.get_loc(col)
iindex = self.index.get_loc(index)
self._mgr.column_setitem(icol, iindex, value, inplace_only=True)
- self._clear_item_cache()
except (KeyError, TypeError, ValueError, LossySetitemError):
# get_loc might raise a KeyError for missing labels (falling back
@@ -4533,7 +4338,6 @@ def _set_value(
self.iloc[index, col] = value
else:
self.loc[index, col] = value
- self._item_cache.pop(col, None)
except InvalidIndexError as ii_err:
# GH48729: Seems like you are trying to assign a value to a
@@ -4565,77 +4369,37 @@ def _ensure_valid_index(self, value) -> None:
self._mgr = self._mgr.reindex_axis(index_copy, axis=1, fill_value=np.nan)
- def _box_col_values(self, values: SingleDataManager, loc: int) -> Series:
+ def _box_col_values(self, values: SingleBlockManager, loc: int) -> Series:
"""
Provide boxed values for a column.
"""
# Lookup in columns so that if e.g. a str datetime was passed
# we attach the Timestamp object as the name.
name = self.columns[loc]
- # We get index=self.index bc values is a SingleDataManager
+ # We get index=self.index bc values is a SingleBlockManager
obj = self._constructor_sliced_from_mgr(values, axes=values.axes)
obj._name = name
return obj.__finalize__(self)
- # ----------------------------------------------------------------------
- # Lookup Caching
-
- def _clear_item_cache(self) -> None:
- self._item_cache.clear()
-
- def _get_item_cache(self, item: Hashable) -> Series:
- """Return the cached item, item represents a label indexer."""
- if using_copy_on_write() or warn_copy_on_write():
- loc = self.columns.get_loc(item)
- return self._ixs(loc, axis=1)
-
- cache = self._item_cache
- res = cache.get(item)
- if res is None:
- # All places that call _get_item_cache have unique columns,
- # pending resolution of GH#33047
-
- loc = self.columns.get_loc(item)
- res = self._ixs(loc, axis=1)
-
- cache[item] = res
-
- # for a chain
- res._is_copy = self._is_copy
- return res
-
- def _reset_cacher(self) -> None:
- # no-op for DataFrame
- pass
-
- def _maybe_cache_changed(self, item, value: Series, inplace: bool) -> None:
- """
- The object has called back to us saying maybe it has changed.
- """
- loc = self._info_axis.get_loc(item)
- arraylike = value._values
-
- old = self._ixs(loc, axis=1)
- if old._values is value._values and inplace:
- # GH#46149 avoid making unnecessary copies/block-splitting
- return
-
- self._mgr.iset(loc, arraylike, inplace=inplace)
+ def _get_item(self, item: Hashable) -> Series:
+ loc = self.columns.get_loc(item)
+ return self._ixs(loc, axis=1)
# ----------------------------------------------------------------------
# Unsorted
@overload
- def query(self, expr: str, *, inplace: Literal[False] = ..., **kwargs) -> DataFrame:
- ...
+ def query(
+ self, expr: str, *, inplace: Literal[False] = ..., **kwargs
+ ) -> DataFrame: ...
@overload
- def query(self, expr: str, *, inplace: Literal[True], **kwargs) -> None:
- ...
+ def query(self, expr: str, *, inplace: Literal[True], **kwargs) -> None: ...
@overload
- def query(self, expr: str, *, inplace: bool = ..., **kwargs) -> DataFrame | None:
- ...
+ def query(
+ self, expr: str, *, inplace: bool = ..., **kwargs
+ ) -> DataFrame | None: ...
def query(self, expr: str, *, inplace: bool = False, **kwargs) -> DataFrame | None:
"""
@@ -4742,9 +4506,9 @@ def query(self, expr: str, *, inplace: bool = False, **kwargs) -> DataFrame | No
Examples
--------
- >>> df = pd.DataFrame({'A': range(1, 6),
- ... 'B': range(10, 0, -2),
- ... 'C C': range(10, 5, -1)})
+ >>> df = pd.DataFrame(
+ ... {"A": range(1, 6), "B": range(10, 0, -2), "C C": range(10, 5, -1)}
+ ... )
>>> df
A B C C
0 1 10 10
@@ -4752,7 +4516,7 @@ def query(self, expr: str, *, inplace: bool = False, **kwargs) -> DataFrame | No
2 3 6 8
3 4 4 7
4 5 2 6
- >>> df.query('A > B')
+ >>> df.query("A > B")
A B C C
4 5 2 6
@@ -4764,13 +4528,13 @@ def query(self, expr: str, *, inplace: bool = False, **kwargs) -> DataFrame | No
For columns with spaces in their name, you can use backtick quoting.
- >>> df.query('B == `C C`')
+ >>> df.query("B == `C C`")
A B C C
0 1 10 10
The previous expression is equivalent to
- >>> df[df.B == df['C C']]
+ >>> df[df.B == df["C C"]]
A B C C
0 1 10 10
"""
@@ -4796,12 +4560,10 @@ def query(self, expr: str, *, inplace: bool = False, **kwargs) -> DataFrame | No
return result
@overload
- def eval(self, expr: str, *, inplace: Literal[False] = ..., **kwargs) -> Any:
- ...
+ def eval(self, expr: str, *, inplace: Literal[False] = ..., **kwargs) -> Any: ...
@overload
- def eval(self, expr: str, *, inplace: Literal[True], **kwargs) -> None:
- ...
+ def eval(self, expr: str, *, inplace: Literal[True], **kwargs) -> None: ...
def eval(self, expr: str, *, inplace: bool = False, **kwargs) -> Any | None:
"""
@@ -4846,7 +4608,7 @@ def eval(self, expr: str, *, inplace: bool = False, **kwargs) -> Any | None:
Examples
--------
- >>> df = pd.DataFrame({'A': range(1, 6), 'B': range(10, 0, -2)})
+ >>> df = pd.DataFrame({"A": range(1, 6), "B": range(10, 0, -2)})
>>> df
A B
0 1 10
@@ -4854,7 +4616,7 @@ def eval(self, expr: str, *, inplace: bool = False, **kwargs) -> Any | None:
2 3 6
3 4 4
4 5 2
- >>> df.eval('A + B')
+ >>> df.eval("A + B")
0 11
1 10
2 9
@@ -4865,7 +4627,7 @@ def eval(self, expr: str, *, inplace: bool = False, **kwargs) -> Any | None:
Assignment is allowed though by default the original DataFrame is not
modified.
- >>> df.eval('C = A + B')
+ >>> df.eval("C = A + B")
A B C
0 1 10 11
1 2 8 10
@@ -4908,7 +4670,7 @@ def eval(self, expr: str, *, inplace: bool = False, **kwargs) -> Any | None:
return _eval(expr, inplace=inplace, **kwargs)
- def select_dtypes(self, include=None, exclude=None) -> Self:
+ def select_dtypes(self, include=None, exclude=None) -> DataFrame:
"""
Return a subset of the DataFrame's columns based on the column dtypes.
@@ -4952,9 +4714,9 @@ def select_dtypes(self, include=None, exclude=None) -> Self:
Examples
--------
- >>> df = pd.DataFrame({'a': [1, 2] * 3,
- ... 'b': [True, False] * 3,
- ... 'c': [1.0, 2.0] * 3})
+ >>> df = pd.DataFrame(
+ ... {"a": [1, 2] * 3, "b": [True, False] * 3, "c": [1.0, 2.0] * 3}
+ ... )
>>> df
a b c
0 1 True 1.0
@@ -4964,7 +4726,7 @@ def select_dtypes(self, include=None, exclude=None) -> Self:
4 1 True 1.0
5 2 False 2.0
- >>> df.select_dtypes(include='bool')
+ >>> df.select_dtypes(include="bool")
b
0 True
1 False
@@ -4973,7 +4735,7 @@ def select_dtypes(self, include=None, exclude=None) -> Self:
4 True
5 False
- >>> df.select_dtypes(include=['float64'])
+ >>> df.select_dtypes(include=["float64"])
c
0 1.0
1 2.0
@@ -4982,7 +4744,7 @@ def select_dtypes(self, include=None, exclude=None) -> Self:
4 1.0
5 2.0
- >>> df.select_dtypes(exclude=['int64'])
+ >>> df.select_dtypes(exclude=["int64"])
b c
0 True 1.0
1 False 2.0
@@ -5048,14 +4810,14 @@ def predicate(arr: ArrayLike) -> bool:
return True
- mgr = self._mgr._get_data_subset(predicate).copy(deep=None)
+ mgr = self._mgr._get_data_subset(predicate).copy(deep=False)
return self._constructor_from_mgr(mgr, axes=mgr.axes).__finalize__(self)
def insert(
self,
loc: int,
column: Hashable,
- value: Scalar | AnyArrayLike,
+ value: object,
allow_duplicates: bool | lib.NoDefault = lib.no_default,
) -> None:
"""
@@ -5081,7 +4843,7 @@ def insert(
Examples
--------
- >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
+ >>> df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4]})
>>> df
col1 col2
0 1 3
@@ -5161,8 +4923,7 @@ def assign(self, **kwargs) -> DataFrame:
Examples
--------
- >>> df = pd.DataFrame({'temp_c': [17.0, 25.0]},
- ... index=['Portland', 'Berkeley'])
+ >>> df = pd.DataFrame({"temp_c": [17.0, 25.0]}, index=["Portland", "Berkeley"])
>>> df
temp_c
Portland 17.0
@@ -5178,7 +4939,7 @@ def assign(self, **kwargs) -> DataFrame:
Alternatively, the same behavior can be achieved by directly
referencing an existing Series or sequence:
- >>> df.assign(temp_f=df['temp_c'] * 9 / 5 + 32)
+ >>> df.assign(temp_f=df["temp_c"] * 9 / 5 + 32)
temp_c temp_f
Portland 17.0 62.6
Berkeley 25.0 77.0
@@ -5186,13 +4947,15 @@ def assign(self, **kwargs) -> DataFrame:
You can create multiple columns within the same assign where one
of the columns depends on another one defined within the same assign:
- >>> df.assign(temp_f=lambda x: x['temp_c'] * 9 / 5 + 32,
- ... temp_k=lambda x: (x['temp_f'] + 459.67) * 5 / 9)
+ >>> df.assign(
+ ... temp_f=lambda x: x["temp_c"] * 9 / 5 + 32,
+ ... temp_k=lambda x: (x["temp_f"] + 459.67) * 5 / 9,
+ ... )
temp_c temp_f temp_k
Portland 17.0 62.6 290.15
Berkeley 25.0 77.0 298.15
"""
- data = self.copy(deep=None)
+ data = self.copy(deep=False)
for k, v in kwargs.items():
data[k] = com.apply_if_callable(v, data)
@@ -5247,9 +5010,7 @@ def _series(self):
# ----------------------------------------------------------------------
# Reindexing and alignment
- def _reindex_multi(
- self, axes: dict[str, Index], copy: bool, fill_value
- ) -> DataFrame:
+ def _reindex_multi(self, axes: dict[str, Index], fill_value) -> DataFrame:
"""
We are guaranteed non-Nones in the axes.
"""
@@ -5271,7 +5032,6 @@ def _reindex_multi(
else:
return self._reindex_with_indexers(
{0: [new_index, row_indexer], 1: [new_columns, col_indexer]},
- copy=copy,
fill_value=fill_value,
)
@@ -5311,7 +5071,7 @@ def set_axis(
labels,
*,
axis: Axis = 0,
- copy: bool | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
) -> DataFrame:
return super().set_axis(labels, axis=axis, copy=copy)
@@ -5328,7 +5088,7 @@ def reindex(
columns=None,
axis: Axis | None = None,
method: ReindexMethod | None = None,
- copy: bool | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
level: Level | None = None,
fill_value: Scalar | None = np.nan,
limit: int | None = None,
@@ -5340,11 +5100,11 @@ def reindex(
columns=columns,
axis=axis,
method=method,
- copy=copy,
level=level,
fill_value=fill_value,
limit=limit,
tolerance=tolerance,
+ copy=copy,
)
@overload
@@ -5358,8 +5118,7 @@ def drop(
level: Level = ...,
inplace: Literal[True],
errors: IgnoreRaise = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def drop(
@@ -5372,8 +5131,7 @@ def drop(
level: Level = ...,
inplace: Literal[False] = ...,
errors: IgnoreRaise = ...,
- ) -> DataFrame:
- ...
+ ) -> DataFrame: ...
@overload
def drop(
@@ -5386,8 +5144,7 @@ def drop(
level: Level = ...,
inplace: bool = ...,
errors: IgnoreRaise = ...,
- ) -> DataFrame | None:
- ...
+ ) -> DataFrame | None: ...
def drop(
self,
@@ -5454,8 +5211,7 @@ def drop(
Examples
--------
- >>> df = pd.DataFrame(np.arange(12).reshape(3, 4),
- ... columns=['A', 'B', 'C', 'D'])
+ >>> df = pd.DataFrame(np.arange(12).reshape(3, 4), columns=["A", "B", "C", "D"])
>>> df
A B C D
0 0 1 2 3
@@ -5464,13 +5220,13 @@ def drop(
Drop columns
- >>> df.drop(['B', 'C'], axis=1)
+ >>> df.drop(["B", "C"], axis=1)
A D
0 0 3
1 4 7
2 8 11
- >>> df.drop(columns=['B', 'C'])
+ >>> df.drop(columns=["B", "C"])
A D
0 0 3
1 4 7
@@ -5484,14 +5240,25 @@ def drop(
Drop columns and/or rows of MultiIndex DataFrame
- >>> midx = pd.MultiIndex(levels=[['llama', 'cow', 'falcon'],
- ... ['speed', 'weight', 'length']],
- ... codes=[[0, 0, 0, 1, 1, 1, 2, 2, 2],
- ... [0, 1, 2, 0, 1, 2, 0, 1, 2]])
- >>> df = pd.DataFrame(index=midx, columns=['big', 'small'],
- ... data=[[45, 30], [200, 100], [1.5, 1], [30, 20],
- ... [250, 150], [1.5, 0.8], [320, 250],
- ... [1, 0.8], [0.3, 0.2]])
+ >>> midx = pd.MultiIndex(
+ ... levels=[["llama", "cow", "falcon"], ["speed", "weight", "length"]],
+ ... codes=[[0, 0, 0, 1, 1, 1, 2, 2, 2], [0, 1, 2, 0, 1, 2, 0, 1, 2]],
+ ... )
+ >>> df = pd.DataFrame(
+ ... index=midx,
+ ... columns=["big", "small"],
+ ... data=[
+ ... [45, 30],
+ ... [200, 100],
+ ... [1.5, 1],
+ ... [30, 20],
+ ... [250, 150],
+ ... [1.5, 0.8],
+ ... [320, 250],
+ ... [1, 0.8],
+ ... [0.3, 0.2],
+ ... ],
+ ... )
>>> df
big small
llama speed 45.0 30.0
@@ -5508,7 +5275,7 @@ def drop(
DataFrame, i.e., drop the combination ``'falcon'`` and
``'weight'``, which deletes only the corresponding row
- >>> df.drop(index=('falcon', 'weight'))
+ >>> df.drop(index=("falcon", "weight"))
big small
llama speed 45.0 30.0
weight 200.0 100.0
@@ -5519,7 +5286,7 @@ def drop(
falcon speed 320.0 250.0
length 0.3 0.2
- >>> df.drop(index='cow', columns='small')
+ >>> df.drop(index="cow", columns="small")
big
llama speed 45.0
weight 200.0
@@ -5528,7 +5295,7 @@ def drop(
weight 1.0
length 0.3
- >>> df.drop(index='length', level=1)
+ >>> df.drop(index="length", level=1)
big small
llama speed 45.0 30.0
weight 200.0 100.0
@@ -5555,12 +5322,11 @@ def rename(
index: Renamer | None = ...,
columns: Renamer | None = ...,
axis: Axis | None = ...,
- copy: bool | None = ...,
+ copy: bool | lib.NoDefault = lib.no_default,
inplace: Literal[True],
level: Level = ...,
errors: IgnoreRaise = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def rename(
@@ -5570,12 +5336,11 @@ def rename(
index: Renamer | None = ...,
columns: Renamer | None = ...,
axis: Axis | None = ...,
- copy: bool | None = ...,
+ copy: bool | lib.NoDefault = lib.no_default,
inplace: Literal[False] = ...,
level: Level = ...,
errors: IgnoreRaise = ...,
- ) -> DataFrame:
- ...
+ ) -> DataFrame: ...
@overload
def rename(
@@ -5585,12 +5350,11 @@ def rename(
index: Renamer | None = ...,
columns: Renamer | None = ...,
axis: Axis | None = ...,
- copy: bool | None = ...,
+ copy: bool | lib.NoDefault = lib.no_default,
inplace: bool = ...,
level: Level = ...,
errors: IgnoreRaise = ...,
- ) -> DataFrame | None:
- ...
+ ) -> DataFrame | None: ...
def rename(
self,
@@ -5599,7 +5363,7 @@ def rename(
index: Renamer | None = None,
columns: Renamer | None = None,
axis: Axis | None = None,
- copy: bool | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
inplace: bool = False,
level: Level | None = None,
errors: IgnoreRaise = "ignore",
@@ -5629,7 +5393,7 @@ def rename(
axis : {0 or 'index', 1 or 'columns'}, default 0
Axis to target with ``mapper``. Can be either the axis name
('index', 'columns') or number (0, 1). The default is 'index'.
- copy : bool, default True
+ copy : bool, default False
Also copy underlying data.
.. note::
@@ -5643,6 +5407,8 @@ def rename(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+
+ .. deprecated:: 3.0.0
inplace : bool, default False
Whether to modify the DataFrame rather than creating a new one.
If True then value of copy is ignored.
@@ -5711,24 +5477,24 @@ def rename(
Using axis-style parameters:
- >>> df.rename(str.lower, axis='columns')
+ >>> df.rename(str.lower, axis="columns")
a b
0 1 4
1 2 5
2 3 6
- >>> df.rename({1: 2, 2: 4}, axis='index')
+ >>> df.rename({1: 2, 2: 4}, axis="index")
A B
0 1 4
2 2 5
4 3 6
"""
+ self._check_copy_deprecation(copy)
return super()._rename(
mapper=mapper,
index=index,
columns=columns,
axis=axis,
- copy=copy,
inplace=inplace,
level=level,
errors=errors,
@@ -5736,7 +5502,7 @@ def rename(
def pop(self, item: Hashable) -> Series:
"""
- Return item and drop from frame. Raise KeyError if not found.
+ Return item and drop it from DataFrame. Raise KeyError if not found.
Parameters
----------
@@ -5746,14 +5512,19 @@ def pop(self, item: Hashable) -> Series:
Returns
-------
Series
+ Series representing the item that is dropped.
Examples
--------
- >>> df = pd.DataFrame([('falcon', 'bird', 389.0),
- ... ('parrot', 'bird', 24.0),
- ... ('lion', 'mammal', 80.5),
- ... ('monkey', 'mammal', np.nan)],
- ... columns=('name', 'class', 'max_speed'))
+ >>> df = pd.DataFrame(
+ ... [
+ ... ("falcon", "bird", 389.0),
+ ... ("parrot", "bird", 24.0),
+ ... ("lion", "mammal", 80.5),
+ ... ("monkey", "mammal", np.nan),
+ ... ],
+ ... columns=("name", "class", "max_speed"),
+ ... )
>>> df
name class max_speed
0 falcon bird 389.0
@@ -5761,7 +5532,7 @@ def pop(self, item: Hashable) -> Series:
2 lion mammal 80.5
3 monkey mammal NaN
- >>> df.pop('class')
+ >>> df.pop("class")
0 bird
1 bird
2 mammal
@@ -5777,9 +5548,19 @@ def pop(self, item: Hashable) -> Series:
"""
return super().pop(item=item)
+ @overload
def _replace_columnwise(
- self, mapping: dict[Hashable, tuple[Any, Any]], inplace: bool, regex
- ):
+ self, mapping: dict[Hashable, tuple[Any, Any]], inplace: Literal[True], regex
+ ) -> None: ...
+
+ @overload
+ def _replace_columnwise(
+ self, mapping: dict[Hashable, tuple[Any, Any]], inplace: Literal[False], regex
+ ) -> Self: ...
+
+ def _replace_columnwise(
+ self, mapping: dict[Hashable, tuple[Any, Any]], inplace: bool, regex
+ ) -> Self | None:
"""
Dispatch to Series.replace column-wise.
@@ -5795,7 +5576,7 @@ def _replace_columnwise(
DataFrame or None
"""
# Operate column-wise
- res = self if inplace else self.copy(deep=None)
+ res = self if inplace else self.copy(deep=False)
ax = self.columns
for i, ax_value in enumerate(ax):
@@ -5808,7 +5589,7 @@ def _replace_columnwise(
res._iset_item(i, newobj, inplace=inplace)
if inplace:
- return
+ return None
return res.__finalize__(self)
@doc(NDFrame.shift, klass=_shared_doc_kwargs["klass"])
@@ -5822,14 +5603,12 @@ def shift(
) -> DataFrame:
if freq is not None and fill_value is not lib.no_default:
# GH#53832
- warnings.warn(
- "Passing a 'freq' together with a 'fill_value' silently ignores "
- "the fill_value and is deprecated. This will raise in a future "
- "version.",
- FutureWarning,
- stacklevel=find_stack_level(),
+ raise ValueError(
+ "Passing a 'freq' together with a 'fill_value' is not allowed."
)
- fill_value = lib.no_default
+
+ if self.empty:
+ return self.copy()
axis = self._get_axis_number(axis)
@@ -5931,8 +5710,7 @@ def set_index(
append: bool = ...,
inplace: Literal[False] = ...,
verify_integrity: bool = ...,
- ) -> DataFrame:
- ...
+ ) -> DataFrame: ...
@overload
def set_index(
@@ -5943,8 +5721,7 @@ def set_index(
append: bool = ...,
inplace: Literal[True],
verify_integrity: bool = ...,
- ) -> None:
- ...
+ ) -> None: ...
def set_index(
self,
@@ -5994,9 +5771,13 @@ def set_index(
Examples
--------
- >>> df = pd.DataFrame({'month': [1, 4, 7, 10],
- ... 'year': [2012, 2014, 2013, 2014],
- ... 'sale': [55, 40, 84, 31]})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "month": [1, 4, 7, 10],
+ ... "year": [2012, 2014, 2013, 2014],
+ ... "sale": [55, 40, 84, 31],
+ ... }
+ ... )
>>> df
month year sale
0 1 2012 55
@@ -6006,7 +5787,7 @@ def set_index(
Set the index to become the 'month' column:
- >>> df.set_index('month')
+ >>> df.set_index("month")
year sale
month
1 2012 55
@@ -6016,7 +5797,7 @@ def set_index(
Create a MultiIndex using columns 'year' and 'month':
- >>> df.set_index(['year', 'month'])
+ >>> df.set_index(["year", "month"])
sale
year month
2012 1 55
@@ -6026,7 +5807,7 @@ def set_index(
Create a MultiIndex using an Index and a column:
- >>> df.set_index([pd.Index([1, 2, 3, 4]), 'year'])
+ >>> df.set_index([pd.Index([1, 2, 3, 4]), "year"])
month sale
year
1 2012 1 55
@@ -6080,8 +5861,7 @@ def set_index(
if inplace:
frame = self
else:
- # GH 49473 Use "lazy copy" with Copy-on-Write
- frame = self.copy(deep=None)
+ frame = self.copy(deep=False)
arrays: list[Index] = []
names: list[Hashable] = []
@@ -6161,8 +5941,7 @@ def reset_index(
col_fill: Hashable = ...,
allow_duplicates: bool | lib.NoDefault = ...,
names: Hashable | Sequence[Hashable] | None = None,
- ) -> DataFrame:
- ...
+ ) -> DataFrame: ...
@overload
def reset_index(
@@ -6175,8 +5954,7 @@ def reset_index(
col_fill: Hashable = ...,
allow_duplicates: bool | lib.NoDefault = ...,
names: Hashable | Sequence[Hashable] | None = None,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def reset_index(
@@ -6189,8 +5967,7 @@ def reset_index(
col_fill: Hashable = ...,
allow_duplicates: bool | lib.NoDefault = ...,
names: Hashable | Sequence[Hashable] | None = None,
- ) -> DataFrame | None:
- ...
+ ) -> DataFrame | None: ...
def reset_index(
self,
@@ -6234,8 +6011,8 @@ def reset_index(
names : int, str or 1-dimensional list, default None
Using the given string, rename the DataFrame column which contains the
- index data. If the DataFrame has a MultiIndex, this has to be a list or
- tuple with length equal to the number of levels.
+ index data. If the DataFrame has a MultiIndex, this has to be a list
+ with length equal to the number of levels.
.. versionadded:: 1.5.0
@@ -6252,12 +6029,11 @@ def reset_index(
Examples
--------
- >>> df = pd.DataFrame([('bird', 389.0),
- ... ('bird', 24.0),
- ... ('mammal', 80.5),
- ... ('mammal', np.nan)],
- ... index=['falcon', 'parrot', 'lion', 'monkey'],
- ... columns=('class', 'max_speed'))
+ >>> df = pd.DataFrame(
+ ... [("bird", 389.0), ("bird", 24.0), ("mammal", 80.5), ("mammal", np.nan)],
+ ... index=["falcon", "parrot", "lion", "monkey"],
+ ... columns=("class", "max_speed"),
+ ... )
>>> df
class max_speed
falcon bird 389.0
@@ -6287,19 +6063,21 @@ class max_speed
You can also use `reset_index` with `MultiIndex`.
- >>> index = pd.MultiIndex.from_tuples([('bird', 'falcon'),
- ... ('bird', 'parrot'),
- ... ('mammal', 'lion'),
- ... ('mammal', 'monkey')],
- ... names=['class', 'name'])
- >>> columns = pd.MultiIndex.from_tuples([('speed', 'max'),
- ... ('species', 'type')])
- >>> df = pd.DataFrame([(389.0, 'fly'),
- ... (24.0, 'fly'),
- ... (80.5, 'run'),
- ... (np.nan, 'jump')],
- ... index=index,
- ... columns=columns)
+ >>> index = pd.MultiIndex.from_tuples(
+ ... [
+ ... ("bird", "falcon"),
+ ... ("bird", "parrot"),
+ ... ("mammal", "lion"),
+ ... ("mammal", "monkey"),
+ ... ],
+ ... names=["class", "name"],
+ ... )
+ >>> columns = pd.MultiIndex.from_tuples([("speed", "max"), ("species", "type")])
+ >>> df = pd.DataFrame(
+ ... [(389.0, "fly"), (24.0, "fly"), (80.5, "run"), (np.nan, "jump")],
+ ... index=index,
+ ... columns=columns,
+ ... )
>>> df
speed species
max type
@@ -6311,7 +6089,7 @@ class name
Using the `names` parameter, choose a name for the index column:
- >>> df.reset_index(names=['classes', 'names'])
+ >>> df.reset_index(names=["classes", "names"])
classes names speed species
max type
0 bird falcon 389.0 fly
@@ -6321,7 +6099,7 @@ class name
If the index has multiple levels, we can reset a subset of them:
- >>> df.reset_index(level='class')
+ >>> df.reset_index(level="class")
class speed species
max type
name
@@ -6333,7 +6111,7 @@ class speed species
If we are not dropping the index, by default, it is placed in the top
level. We can place it in another level:
- >>> df.reset_index(level='class', col_level=1)
+ >>> df.reset_index(level="class", col_level=1)
speed species
class max type
name
@@ -6345,7 +6123,7 @@ class max type
When the index is inserted under another level, we can specify under
which one with the parameter `col_fill`:
- >>> df.reset_index(level='class', col_level=1, col_fill='species')
+ >>> df.reset_index(level="class", col_level=1, col_fill="species")
species speed species
class max type
name
@@ -6356,7 +6134,7 @@ class max type
If we specify a nonexistent level for `col_fill`, it is created:
- >>> df.reset_index(level='class', col_level=1, col_fill='genus')
+ >>> df.reset_index(level="class", col_level=1, col_fill="genus")
genus speed species
class max type
name
@@ -6370,7 +6148,7 @@ class max type
if inplace:
new_obj = self
else:
- new_obj = self.copy(deep=None)
+ new_obj = self.copy(deep=False)
if allow_duplicates is not lib.no_default:
allow_duplicates = validate_bool_kwarg(allow_duplicates, "allow_duplicates")
@@ -6475,8 +6253,7 @@ def dropna(
subset: IndexLabel = ...,
inplace: Literal[False] = ...,
ignore_index: bool = ...,
- ) -> DataFrame:
- ...
+ ) -> DataFrame: ...
@overload
def dropna(
@@ -6488,8 +6265,7 @@ def dropna(
subset: IndexLabel = ...,
inplace: Literal[True],
ignore_index: bool = ...,
- ) -> None:
- ...
+ ) -> None: ...
def dropna(
self,
@@ -6497,7 +6273,7 @@ def dropna(
axis: Axis = 0,
how: AnyAll | lib.NoDefault = lib.no_default,
thresh: int | lib.NoDefault = lib.no_default,
- subset: IndexLabel | None = None,
+ subset: IndexLabel | AnyArrayLike | None = None,
inplace: bool = False,
ignore_index: bool = False,
) -> DataFrame | None:
@@ -6552,10 +6328,13 @@ def dropna(
Examples
--------
- >>> df = pd.DataFrame({"name": ['Alfred', 'Batman', 'Catwoman'],
- ... "toy": [np.nan, 'Batmobile', 'Bullwhip'],
- ... "born": [pd.NaT, pd.Timestamp("1940-04-25"),
- ... pd.NaT]})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "name": ["Alfred", "Batman", "Catwoman"],
+ ... "toy": [np.nan, "Batmobile", "Bullwhip"],
+ ... "born": [pd.NaT, pd.Timestamp("1940-04-25"), pd.NaT],
+ ... }
+ ... )
>>> df
name toy born
0 Alfred NaN NaT
@@ -6570,7 +6349,7 @@ def dropna(
Drop the columns where at least one element is missing.
- >>> df.dropna(axis='columns')
+ >>> df.dropna(axis="columns")
name
0 Alfred
1 Batman
@@ -6578,7 +6357,7 @@ def dropna(
Drop the rows where all elements are missing.
- >>> df.dropna(how='all')
+ >>> df.dropna(how="all")
name toy born
0 Alfred NaN NaT
1 Batman Batmobile 1940-04-25
@@ -6593,7 +6372,7 @@ def dropna(
Define in which columns to look for missing values.
- >>> df.dropna(subset=['name', 'toy'])
+ >>> df.dropna(subset=["name", "toy"])
name toy born
1 Batman Batmobile 1940-04-25
2 Catwoman Bullwhip NaT
@@ -6618,7 +6397,7 @@ def dropna(
if subset is not None:
# subset needs to be list
if not is_list_like(subset):
- subset = [subset]
+ subset = [cast(Hashable, subset)]
ax = self._get_axis(agg_axis)
indices = ax.get_indexer_for(subset)
check = indices == -1
@@ -6639,7 +6418,7 @@ def dropna(
raise ValueError(f"invalid how option: {how}")
if np.all(mask):
- result = self.copy(deep=None)
+ result = self.copy(deep=False)
else:
result = self.loc(axis=axis)[mask]
@@ -6659,8 +6438,7 @@ def drop_duplicates(
keep: DropKeep = ...,
inplace: Literal[True],
ignore_index: bool = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def drop_duplicates(
@@ -6670,8 +6448,7 @@ def drop_duplicates(
keep: DropKeep = ...,
inplace: Literal[False] = ...,
ignore_index: bool = ...,
- ) -> DataFrame:
- ...
+ ) -> DataFrame: ...
@overload
def drop_duplicates(
@@ -6681,8 +6458,7 @@ def drop_duplicates(
keep: DropKeep = ...,
inplace: bool = ...,
ignore_index: bool = ...,
- ) -> DataFrame | None:
- ...
+ ) -> DataFrame | None: ...
def drop_duplicates(
self,
@@ -6724,15 +6500,22 @@ def drop_duplicates(
--------
DataFrame.value_counts: Count unique combinations of columns.
+ Notes
+ -----
+ This method requires columns specified by ``subset`` to be of hashable type.
+ Passing unhashable columns will raise a ``TypeError``.
+
Examples
--------
Consider dataset containing ramen rating.
- >>> df = pd.DataFrame({
- ... 'brand': ['Yum Yum', 'Yum Yum', 'Indomie', 'Indomie', 'Indomie'],
- ... 'style': ['cup', 'cup', 'cup', 'pack', 'pack'],
- ... 'rating': [4, 4, 3.5, 15, 5]
- ... })
+ >>> df = pd.DataFrame(
+ ... {
+ ... "brand": ["Yum Yum", "Yum Yum", "Indomie", "Indomie", "Indomie"],
+ ... "style": ["cup", "cup", "cup", "pack", "pack"],
+ ... "rating": [4, 4, 3.5, 15, 5],
+ ... }
+ ... )
>>> df
brand style rating
0 Yum Yum cup 4.0
@@ -6752,21 +6535,21 @@ def drop_duplicates(
To remove duplicates on specific column(s), use ``subset``.
- >>> df.drop_duplicates(subset=['brand'])
+ >>> df.drop_duplicates(subset=["brand"])
brand style rating
0 Yum Yum cup 4.0
2 Indomie cup 3.5
To remove duplicates and keep last occurrences, use ``keep``.
- >>> df.drop_duplicates(subset=['brand', 'style'], keep='last')
+ >>> df.drop_duplicates(subset=["brand", "style"], keep="last")
brand style rating
1 Yum Yum cup 4.0
2 Indomie cup 3.5
4 Indomie pack 5.0
"""
if self.empty:
- return self.copy(deep=None)
+ return self.copy(deep=False)
inplace = validate_bool_kwarg(inplace, "inplace")
ignore_index = validate_bool_kwarg(ignore_index, "ignore_index")
@@ -6819,11 +6602,13 @@ def duplicated(
--------
Consider dataset containing ramen rating.
- >>> df = pd.DataFrame({
- ... 'brand': ['Yum Yum', 'Yum Yum', 'Indomie', 'Indomie', 'Indomie'],
- ... 'style': ['cup', 'cup', 'cup', 'pack', 'pack'],
- ... 'rating': [4, 4, 3.5, 15, 5]
- ... })
+ >>> df = pd.DataFrame(
+ ... {
+ ... "brand": ["Yum Yum", "Yum Yum", "Indomie", "Indomie", "Indomie"],
+ ... "style": ["cup", "cup", "cup", "pack", "pack"],
+ ... "rating": [4, 4, 3.5, 15, 5],
+ ... }
+ ... )
>>> df
brand style rating
0 Yum Yum cup 4.0
@@ -6846,7 +6631,7 @@ def duplicated(
By using 'last', the last occurrence of each set of duplicated values
is set on False and all others on True.
- >>> df.duplicated(keep='last')
+ >>> df.duplicated(keep="last")
0 True
1 False
2 False
@@ -6866,7 +6651,7 @@ def duplicated(
To find duplicates on specific column(s), use ``subset``.
- >>> df.duplicated(subset=['brand'])
+ >>> df.duplicated(subset=["brand"])
0 False
1 True
2 False
@@ -6880,7 +6665,7 @@ def duplicated(
def f(vals) -> tuple[np.ndarray, int]:
labels, shape = algorithms.factorize(vals, size_hint=len(self))
- return labels.astype("i8", copy=False), len(shape)
+ return labels.astype("i8"), len(shape)
if subset is None:
# https://github.com/pandas-dev/pandas/issues/28770
@@ -6932,8 +6717,7 @@ def sort_values(
na_position: NaPosition = ...,
ignore_index: bool = ...,
key: ValueKeyFunc = ...,
- ) -> DataFrame:
- ...
+ ) -> DataFrame: ...
@overload
def sort_values(
@@ -6947,8 +6731,7 @@ def sort_values(
na_position: str = ...,
ignore_index: bool = ...,
key: ValueKeyFunc = ...,
- ) -> None:
- ...
+ ) -> None: ...
def sort_values(
self,
@@ -7012,12 +6795,14 @@ def sort_values(
Examples
--------
- >>> df = pd.DataFrame({
- ... 'col1': ['A', 'A', 'B', np.nan, 'D', 'C'],
- ... 'col2': [2, 1, 9, 8, 7, 4],
- ... 'col3': [0, 1, 9, 4, 2, 3],
- ... 'col4': ['a', 'B', 'c', 'D', 'e', 'F']
- ... })
+ >>> df = pd.DataFrame(
+ ... {
+ ... "col1": ["A", "A", "B", np.nan, "D", "C"],
+ ... "col2": [2, 1, 9, 8, 7, 4],
+ ... "col3": [0, 1, 9, 4, 2, 3],
+ ... "col4": ["a", "B", "c", "D", "e", "F"],
+ ... }
+ ... )
>>> df
col1 col2 col3 col4
0 A 2 0 a
@@ -7027,9 +6812,11 @@ def sort_values(
4 D 7 2 e
5 C 4 3 F
- Sort by col1
+ **Sort by a single column**
- >>> df.sort_values(by=['col1'])
+ In this case, we are sorting the rows according to values in ``col1``:
+
+ >>> df.sort_values(by=["col1"])
col1 col2 col3 col4
0 A 2 0 a
1 A 1 1 B
@@ -7038,9 +6825,14 @@ def sort_values(
4 D 7 2 e
3 NaN 8 4 D
- Sort by multiple columns
+ **Sort by multiple columns**
+
+ You can also provide multiple columns to ``by`` argument, as shown below.
+ In this example, the rows are first sorted according to ``col1``, and then
+ the rows that have an identical value in ``col1`` are sorted according
+ to ``col2``.
- >>> df.sort_values(by=['col1', 'col2'])
+ >>> df.sort_values(by=["col1", "col2"])
col1 col2 col3 col4
1 A 1 1 B
0 A 2 0 a
@@ -7049,9 +6841,11 @@ def sort_values(
4 D 7 2 e
3 NaN 8 4 D
- Sort Descending
+ **Sort in a descending order**
- >>> df.sort_values(by='col1', ascending=False)
+ The sort order can be reversed using ``ascending`` argument, as shown below:
+
+ >>> df.sort_values(by="col1", ascending=False)
col1 col2 col3 col4
4 D 7 2 e
5 C 4 3 F
@@ -7060,9 +6854,13 @@ def sort_values(
1 A 1 1 B
3 NaN 8 4 D
- Putting NAs first
+ **Placing any** ``NA`` **first**
+
+ Note that in the above example, the rows that contain an ``NA`` value in their
+ ``col1`` are placed at the end of the dataframe. This behavior can be modified
+ via ``na_position`` argument, as shown below:
- >>> df.sort_values(by='col1', ascending=False, na_position='first')
+ >>> df.sort_values(by="col1", ascending=False, na_position="first")
col1 col2 col3 col4
3 NaN 8 4 D
4 D 7 2 e
@@ -7071,9 +6869,14 @@ def sort_values(
0 A 2 0 a
1 A 1 1 B
- Sorting with a key function
+ **Customized sort order**
- >>> df.sort_values(by='col4', key=lambda col: col.str.lower())
+ The ``key`` argument allows for a further customization of sorting behaviour.
+ For example, you may want
+ to ignore the `letter's case `__
+ when sorting strings:
+
+ >>> df.sort_values(by="col4", key=lambda col: col.str.lower())
col1 col2 col3 col4
0 A 2 0 a
1 A 1 1 B
@@ -7082,13 +6885,19 @@ def sort_values(
4 D 7 2 e
5 C 4 3 F
- Natural sort with the key argument,
- using the `natsort ` package.
+ Another typical example is
+ `natural sorting `__.
+ This can be done using
+ ``natsort`` `package `__,
+ which provides sorted indices according
+ to their natural order, as shown below:
- >>> df = pd.DataFrame({
- ... "time": ['0hr', '128hr', '72hr', '48hr', '96hr'],
- ... "value": [10, 20, 30, 40, 50]
- ... })
+ >>> df = pd.DataFrame(
+ ... {
+ ... "time": ["0hr", "128hr", "72hr", "48hr", "96hr"],
+ ... "value": [10, 20, 30, 40, 50],
+ ... }
+ ... )
>>> df
time value
0 0hr 10
@@ -7097,9 +6906,11 @@ def sort_values(
3 48hr 40
4 96hr 50
>>> from natsort import index_natsorted
+ >>> index_natsorted(df["time"])
+ [0, 3, 2, 4, 1]
>>> df.sort_values(
... by="time",
- ... key=lambda x: np.argsort(index_natsorted(df["time"]))
+ ... key=lambda x: np.argsort(index_natsorted(x)),
... )
time value
0 0hr 10
@@ -7160,10 +6971,10 @@ def sort_values(
if inplace:
return self._update_inplace(self)
else:
- return self.copy(deep=None)
+ return self.copy(deep=False)
if is_range_indexer(indexer, len(indexer)):
- result = self.copy(deep=(not inplace and not using_copy_on_write()))
+ result = self.copy(deep=False)
if ignore_index:
result.index = default_index(len(result))
@@ -7200,8 +7011,7 @@ def sort_index(
sort_remaining: bool = ...,
ignore_index: bool = ...,
key: IndexKeyFunc = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def sort_index(
@@ -7216,8 +7026,7 @@ def sort_index(
sort_remaining: bool = ...,
ignore_index: bool = ...,
key: IndexKeyFunc = ...,
- ) -> DataFrame:
- ...
+ ) -> DataFrame: ...
@overload
def sort_index(
@@ -7232,8 +7041,7 @@ def sort_index(
sort_remaining: bool = ...,
ignore_index: bool = ...,
key: IndexKeyFunc = ...,
- ) -> DataFrame | None:
- ...
+ ) -> DataFrame | None: ...
def sort_index(
self,
@@ -7300,8 +7108,9 @@ def sort_index(
Examples
--------
- >>> df = pd.DataFrame([1, 2, 3, 4, 5], index=[100, 29, 234, 1, 150],
- ... columns=['A'])
+ >>> df = pd.DataFrame(
+ ... [1, 2, 3, 4, 5], index=[100, 29, 234, 1, 150], columns=["A"]
+ ... )
>>> df.sort_index()
A
1 4
@@ -7324,7 +7133,7 @@ def sort_index(
A key function can be specified which is applied to the index before
sorting. For a ``MultiIndex`` this is applied to each level separately.
- >>> df = pd.DataFrame({"a": [1, 2, 3, 4]}, index=['A', 'b', 'C', 'd'])
+ >>> df = pd.DataFrame({"a": [1, 2, 3, 4]}, index=["A", "b", "C", "d"])
>>> df.sort_index(key=lambda x: x.str.lower())
a
A 1
@@ -7388,9 +7197,10 @@ def value_counts(
Examples
--------
- >>> df = pd.DataFrame({'num_legs': [2, 4, 4, 6],
- ... 'num_wings': [2, 0, 0, 0]},
- ... index=['falcon', 'dog', 'cat', 'ant'])
+ >>> df = pd.DataFrame(
+ ... {"num_legs": [2, 4, 4, 6], "num_wings": [2, 0, 0, 0]},
+ ... index=["falcon", "dog", "cat", "ant"],
+ ... )
>>> df
num_legs num_wings
falcon 2 2
@@ -7428,8 +7238,12 @@ def value_counts(
With `dropna` set to `False` we can also count rows with NA values.
- >>> df = pd.DataFrame({'first_name': ['John', 'Anne', 'John', 'Beth'],
- ... 'middle_name': ['Smith', pd.NA, pd.NA, 'Louise']})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "first_name": ["John", "Anne", "John", "Beth"],
+ ... "middle_name": ["Smith", pd.NA, pd.NA, "Louise"],
+ ... }
+ ... )
>>> df
first_name middle_name
0 John Smith
@@ -7527,16 +7341,34 @@ def nlargest(
Examples
--------
- >>> df = pd.DataFrame({'population': [59000000, 65000000, 434000,
- ... 434000, 434000, 337000, 11300,
- ... 11300, 11300],
- ... 'GDP': [1937894, 2583560 , 12011, 4520, 12128,
- ... 17036, 182, 38, 311],
- ... 'alpha-2': ["IT", "FR", "MT", "MV", "BN",
- ... "IS", "NR", "TV", "AI"]},
- ... index=["Italy", "France", "Malta",
- ... "Maldives", "Brunei", "Iceland",
- ... "Nauru", "Tuvalu", "Anguilla"])
+ >>> df = pd.DataFrame(
+ ... {
+ ... "population": [
+ ... 59000000,
+ ... 65000000,
+ ... 434000,
+ ... 434000,
+ ... 434000,
+ ... 337000,
+ ... 11300,
+ ... 11300,
+ ... 11300,
+ ... ],
+ ... "GDP": [1937894, 2583560, 12011, 4520, 12128, 17036, 182, 38, 311],
+ ... "alpha-2": ["IT", "FR", "MT", "MV", "BN", "IS", "NR", "TV", "AI"],
+ ... },
+ ... index=[
+ ... "Italy",
+ ... "France",
+ ... "Malta",
+ ... "Maldives",
+ ... "Brunei",
+ ... "Iceland",
+ ... "Nauru",
+ ... "Tuvalu",
+ ... "Anguilla",
+ ... ],
+ ... )
>>> df
population GDP alpha-2
Italy 59000000 1937894 IT
@@ -7552,7 +7384,7 @@ def nlargest(
In the following example, we will use ``nlargest`` to select the three
rows having the largest values in column "population".
- >>> df.nlargest(3, 'population')
+ >>> df.nlargest(3, "population")
population GDP alpha-2
France 65000000 2583560 FR
Italy 59000000 1937894 IT
@@ -7560,7 +7392,7 @@ def nlargest(
When using ``keep='last'``, ties are resolved in reverse order:
- >>> df.nlargest(3, 'population', keep='last')
+ >>> df.nlargest(3, "population", keep="last")
population GDP alpha-2
France 65000000 2583560 FR
Italy 59000000 1937894 IT
@@ -7570,7 +7402,7 @@ def nlargest(
if there are duplicate values for the smallest element, all the
ties are kept:
- >>> df.nlargest(3, 'population', keep='all')
+ >>> df.nlargest(3, "population", keep="all")
population GDP alpha-2
France 65000000 2583560 FR
Italy 59000000 1937894 IT
@@ -7580,7 +7412,7 @@ def nlargest(
However, ``nlargest`` does not keep ``n`` distinct largest elements:
- >>> df.nlargest(5, 'population', keep='all')
+ >>> df.nlargest(5, "population", keep="all")
population GDP alpha-2
France 65000000 2583560 FR
Italy 59000000 1937894 IT
@@ -7591,7 +7423,7 @@ def nlargest(
To order by the largest values in column "population" and then "GDP",
we can specify multiple columns like in the next example.
- >>> df.nlargest(3, ['population', 'GDP'])
+ >>> df.nlargest(3, ["population", "GDP"])
population GDP alpha-2
France 65000000 2583560 FR
Italy 59000000 1937894 IT
@@ -7630,6 +7462,7 @@ def nsmallest(
Returns
-------
DataFrame
+ DataFrame with the first `n` rows ordered by `columns` in ascending order.
See Also
--------
@@ -7640,16 +7473,34 @@ def nsmallest(
Examples
--------
- >>> df = pd.DataFrame({'population': [59000000, 65000000, 434000,
- ... 434000, 434000, 337000, 337000,
- ... 11300, 11300],
- ... 'GDP': [1937894, 2583560 , 12011, 4520, 12128,
- ... 17036, 182, 38, 311],
- ... 'alpha-2': ["IT", "FR", "MT", "MV", "BN",
- ... "IS", "NR", "TV", "AI"]},
- ... index=["Italy", "France", "Malta",
- ... "Maldives", "Brunei", "Iceland",
- ... "Nauru", "Tuvalu", "Anguilla"])
+ >>> df = pd.DataFrame(
+ ... {
+ ... "population": [
+ ... 59000000,
+ ... 65000000,
+ ... 434000,
+ ... 434000,
+ ... 434000,
+ ... 337000,
+ ... 337000,
+ ... 11300,
+ ... 11300,
+ ... ],
+ ... "GDP": [1937894, 2583560, 12011, 4520, 12128, 17036, 182, 38, 311],
+ ... "alpha-2": ["IT", "FR", "MT", "MV", "BN", "IS", "NR", "TV", "AI"],
+ ... },
+ ... index=[
+ ... "Italy",
+ ... "France",
+ ... "Malta",
+ ... "Maldives",
+ ... "Brunei",
+ ... "Iceland",
+ ... "Nauru",
+ ... "Tuvalu",
+ ... "Anguilla",
+ ... ],
+ ... )
>>> df
population GDP alpha-2
Italy 59000000 1937894 IT
@@ -7665,7 +7516,7 @@ def nsmallest(
In the following example, we will use ``nsmallest`` to select the
three rows having the smallest values in column "population".
- >>> df.nsmallest(3, 'population')
+ >>> df.nsmallest(3, "population")
population GDP alpha-2
Tuvalu 11300 38 TV
Anguilla 11300 311 AI
@@ -7673,7 +7524,7 @@ def nsmallest(
When using ``keep='last'``, ties are resolved in reverse order:
- >>> df.nsmallest(3, 'population', keep='last')
+ >>> df.nsmallest(3, "population", keep="last")
population GDP alpha-2
Anguilla 11300 311 AI
Tuvalu 11300 38 TV
@@ -7683,7 +7534,7 @@ def nsmallest(
if there are duplicate values for the largest element, all the
ties are kept.
- >>> df.nsmallest(3, 'population', keep='all')
+ >>> df.nsmallest(3, "population", keep="all")
population GDP alpha-2
Tuvalu 11300 38 TV
Anguilla 11300 311 AI
@@ -7693,7 +7544,7 @@ def nsmallest(
However, ``nsmallest`` does not keep ``n`` distinct
smallest elements:
- >>> df.nsmallest(4, 'population', keep='all')
+ >>> df.nsmallest(4, "population", keep="all")
population GDP alpha-2
Tuvalu 11300 38 TV
Anguilla 11300 311 AI
@@ -7703,7 +7554,7 @@ def nsmallest(
To order by the smallest values in column "population" and then "GDP", we can
specify multiple columns like in the next example.
- >>> df.nsmallest(3, ['population', 'GDP'])
+ >>> df.nsmallest(3, ["population", "GDP"])
population GDP alpha-2
Tuvalu 11300 38 TV
Anguilla 11300 311 AI
@@ -7774,7 +7625,7 @@ def nsmallest(
),
)
def swaplevel(self, i: Axis = -2, j: Axis = -1, axis: Axis = 0) -> DataFrame:
- result = self.copy(deep=None)
+ result = self.copy(deep=False)
axis = self._get_axis_number(axis)
@@ -7791,7 +7642,9 @@ def swaplevel(self, i: Axis = -2, j: Axis = -1, axis: Axis = 0) -> DataFrame:
def reorder_levels(self, order: Sequence[int | str], axis: Axis = 0) -> DataFrame:
"""
- Rearrange index levels using input order. May not drop or duplicate levels.
+ Rearrange index or column levels using input ``order``.
+
+ May not drop or duplicate levels.
Parameters
----------
@@ -7804,6 +7657,7 @@ def reorder_levels(self, order: Sequence[int | str], axis: Axis = 0) -> DataFram
Returns
-------
DataFrame
+ DataFrame with indices or columns with reordered levels.
Examples
--------
@@ -7834,7 +7688,7 @@ class diet
if not isinstance(self._get_axis(axis), MultiIndex): # pragma: no cover
raise TypeError("Can only reorder levels on a hierarchical axis.")
- result = self.copy(deep=None)
+ result = self.copy(deep=False)
if axis == 0:
assert isinstance(result.index, MultiIndex)
@@ -7910,13 +7764,7 @@ def _dispatch_frame_op(
# TODO operate_blockwise expects a manager of the same type
bm = self._mgr.operate_blockwise(
- # error: Argument 1 to "operate_blockwise" of "ArrayManager" has
- # incompatible type "Union[ArrayManager, BlockManager]"; expected
- # "ArrayManager"
- # error: Argument 1 to "operate_blockwise" of "BlockManager" has
- # incompatible type "Union[ArrayManager, BlockManager]"; expected
- # "BlockManager"
- right._mgr, # type: ignore[arg-type]
+ right._mgr,
array_op,
)
return self._constructor_from_mgr(bm, axes=bm.axes)
@@ -7984,19 +7832,17 @@ def _arith_method_with_reindex(self, right: DataFrame, op) -> DataFrame:
left = self
# GH#31623, only operate on shared columns
- cols, lcols, rcols = left.columns.join(
- right.columns, how="inner", level=None, return_indexers=True
+ cols, lcol_indexer, rcol_indexer = left.columns.join(
+ right.columns, how="inner", return_indexers=True
)
- new_left = left.iloc[:, lcols]
- new_right = right.iloc[:, rcols]
+ new_left = left if lcol_indexer is None else left.iloc[:, lcol_indexer]
+ new_right = right if rcol_indexer is None else right.iloc[:, rcol_indexer]
result = op(new_left, new_right)
# Do the join on the columns instead of using left._align_for_op
# to avoid constructing two potentially large/sparse DataFrames
- join_columns, _, _ = left.columns.join(
- right.columns, how="outer", level=None, return_indexers=True
- )
+ join_columns = left.columns.join(right.columns, how="outer")
if result.columns.has_duplicates:
# Avoid reindexing with a duplicate axis.
@@ -8145,9 +7991,7 @@ def to_series(right):
if flex is not None and isinstance(right, DataFrame):
if not left._indexed_same(right):
if flex:
- left, right = left.align(
- right, join="outer", level=level, copy=False
- )
+ left, right = left.align(right, join="outer", level=level)
else:
raise ValueError(
"Can only compare identically-labeled (both index and columns) "
@@ -8160,7 +8004,7 @@ def to_series(right):
if not left.axes[axis].equals(right.index):
raise ValueError(
"Operands are not aligned. Do "
- "`left, right = left.align(right, axis=1, copy=False)` "
+ "`left, right = left.align(right, axis=1)` "
"before operating."
)
@@ -8169,7 +8013,6 @@ def to_series(right):
join="outer",
axis=axis,
level=level,
- copy=False,
)
right = left._maybe_align_series_as_frame(right, axis)
@@ -8596,8 +8439,8 @@ def combine(
--------
Combine using a simple function that chooses the smaller column.
- >>> df1 = pd.DataFrame({'A': [0, 0], 'B': [4, 4]})
- >>> df2 = pd.DataFrame({'A': [1, 1], 'B': [3, 3]})
+ >>> df1 = pd.DataFrame({"A": [0, 0], "B": [4, 4]})
+ >>> df2 = pd.DataFrame({"A": [1, 1], "B": [3, 3]})
>>> take_smaller = lambda s1, s2: s1 if s1.sum() < s2.sum() else s2
>>> df1.combine(df2, take_smaller)
A B
@@ -8606,8 +8449,8 @@ def combine(
Example using a true element-wise combine function.
- >>> df1 = pd.DataFrame({'A': [5, 0], 'B': [2, 4]})
- >>> df2 = pd.DataFrame({'A': [1, 1], 'B': [3, 3]})
+ >>> df1 = pd.DataFrame({"A": [5, 0], "B": [2, 4]})
+ >>> df2 = pd.DataFrame({"A": [1, 1], "B": [3, 3]})
>>> df1.combine(df2, np.minimum)
A B
0 1 2
@@ -8616,8 +8459,8 @@ def combine(
Using `fill_value` fills Nones prior to passing the column to the
merge function.
- >>> df1 = pd.DataFrame({'A': [0, 0], 'B': [None, 4]})
- >>> df2 = pd.DataFrame({'A': [1, 1], 'B': [3, 3]})
+ >>> df1 = pd.DataFrame({"A": [0, 0], "B": [None, 4]})
+ >>> df2 = pd.DataFrame({"A": [1, 1], "B": [3, 3]})
>>> df1.combine(df2, take_smaller, fill_value=-5)
A B
0 0 -5.0
@@ -8626,8 +8469,8 @@ def combine(
However, if the same element in both dataframes is None, that None
is preserved
- >>> df1 = pd.DataFrame({'A': [0, 0], 'B': [None, 4]})
- >>> df2 = pd.DataFrame({'A': [1, 1], 'B': [None, 3]})
+ >>> df1 = pd.DataFrame({"A": [0, 0], "B": [None, 4]})
+ >>> df2 = pd.DataFrame({"A": [1, 1], "B": [None, 3]})
>>> df1.combine(df2, take_smaller, fill_value=-5)
A B
0 0 -5.0
@@ -8636,8 +8479,14 @@ def combine(
Example that demonstrates the use of `overwrite` and behavior when
the axis differ between the dataframes.
- >>> df1 = pd.DataFrame({'A': [0, 0], 'B': [4, 4]})
- >>> df2 = pd.DataFrame({'B': [3, 3], 'C': [-10, 1], }, index=[1, 2])
+ >>> df1 = pd.DataFrame({"A": [0, 0], "B": [4, 4]})
+ >>> df2 = pd.DataFrame(
+ ... {
+ ... "B": [3, 3],
+ ... "C": [-10, 1],
+ ... },
+ ... index=[1, 2],
+ ... )
>>> df1.combine(df2, take_smaller)
A B C
0 NaN NaN NaN
@@ -8652,7 +8501,13 @@ def combine(
Demonstrating the preference of the passed in dataframe.
- >>> df2 = pd.DataFrame({'B': [3, 3], 'C': [1, 1], }, index=[1, 2])
+ >>> df2 = pd.DataFrame(
+ ... {
+ ... "B": [3, 3],
+ ... "C": [1, 1],
+ ... },
+ ... index=[1, 2],
+ ... )
>>> df2.combine(df1, take_smaller)
A B C
0 0.0 NaN NaN
@@ -8667,7 +8522,7 @@ def combine(
"""
other_idxlen = len(other.index) # save for compare
- this, other = self.align(other, copy=False)
+ this, other = self.align(other)
new_index = this.index
if other.empty and len(new_index) == len(self.index):
@@ -8707,15 +8562,15 @@ def combine(
# try to promote series, which is all NaN, as other_dtype.
new_dtype = other_dtype
try:
- series = series.astype(new_dtype, copy=False)
+ series = series.astype(new_dtype)
except ValueError:
# e.g. new_dtype is integer types
pass
else:
# if we have different dtypes, possibly promote
new_dtype = find_common_type([this_dtype, other_dtype])
- series = series.astype(new_dtype, copy=False)
- other_series = other_series.astype(new_dtype, copy=False)
+ series = series.astype(new_dtype)
+ other_series = other_series.astype(new_dtype)
arr = func(series, other_series)
if isinstance(new_dtype, np.dtype):
@@ -8762,8 +8617,8 @@ def combine_first(self, other: DataFrame) -> DataFrame:
Examples
--------
- >>> df1 = pd.DataFrame({'A': [None, 0], 'B': [None, 4]})
- >>> df2 = pd.DataFrame({'A': [1, 1], 'B': [3, 3]})
+ >>> df1 = pd.DataFrame({"A": [None, 0], "B": [None, 4]})
+ >>> df2 = pd.DataFrame({"A": [1, 1], "B": [3, 3]})
>>> df1.combine_first(df2)
A B
0 1.0 3.0
@@ -8772,8 +8627,8 @@ def combine_first(self, other: DataFrame) -> DataFrame:
Null values still persist if the location of that null value
does not exist in `other`
- >>> df1 = pd.DataFrame({'A': [None, 0], 'B': [4, None]})
- >>> df2 = pd.DataFrame({'B': [3, 3], 'C': [1, 1]}, index=[1, 2])
+ >>> df1 = pd.DataFrame({"A": [None, 0], "B": [4, None]})
+ >>> df2 = pd.DataFrame({"B": [3, 3], "C": [1, 1]}, index=[1, 2])
>>> df1.combine_first(df2)
A B C
0 NaN 4.0 NaN
@@ -8870,12 +8725,14 @@ def update(
dict.update : Similar method for dictionaries.
DataFrame.merge : For column(s)-on-column(s) operations.
+ Notes
+ -----
+ 1. Duplicate indices on `other` are not supported and raises `ValueError`.
+
Examples
--------
- >>> df = pd.DataFrame({'A': [1, 2, 3],
- ... 'B': [400, 500, 600]})
- >>> new_df = pd.DataFrame({'B': [4, 5, 6],
- ... 'C': [7, 8, 9]})
+ >>> df = pd.DataFrame({"A": [1, 2, 3], "B": [400, 500, 600]})
+ >>> new_df = pd.DataFrame({"B": [4, 5, 6], "C": [7, 8, 9]})
>>> df.update(new_df)
>>> df
A B
@@ -8886,9 +8743,8 @@ def update(
The DataFrame's length does not increase as a result of the update,
only values at matching index/column labels are updated.
- >>> df = pd.DataFrame({'A': ['a', 'b', 'c'],
- ... 'B': ['x', 'y', 'z']})
- >>> new_df = pd.DataFrame({'B': ['d', 'e', 'f', 'g', 'h', 'i']})
+ >>> df = pd.DataFrame({"A": ["a", "b", "c"], "B": ["x", "y", "z"]})
+ >>> new_df = pd.DataFrame({"B": ["d", "e", "f", "g", "h", "i"]})
>>> df.update(new_df)
>>> df
A B
@@ -8896,9 +8752,8 @@ def update(
1 b e
2 c f
- >>> df = pd.DataFrame({'A': ['a', 'b', 'c'],
- ... 'B': ['x', 'y', 'z']})
- >>> new_df = pd.DataFrame({'B': ['d', 'f']}, index=[0, 2])
+ >>> df = pd.DataFrame({"A": ["a", "b", "c"], "B": ["x", "y", "z"]})
+ >>> new_df = pd.DataFrame({"B": ["d", "f"]}, index=[0, 2])
>>> df.update(new_df)
>>> df
A B
@@ -8908,9 +8763,8 @@ def update(
For Series, its name attribute must be set.
- >>> df = pd.DataFrame({'A': ['a', 'b', 'c'],
- ... 'B': ['x', 'y', 'z']})
- >>> new_column = pd.Series(['d', 'e', 'f'], name='B')
+ >>> df = pd.DataFrame({"A": ["a", "b", "c"], "B": ["x", "y", "z"]})
+ >>> new_column = pd.Series(["d", "e", "f"], name="B")
>>> df.update(new_column)
>>> df
A B
@@ -8921,9 +8775,8 @@ def update(
If `other` contains NaNs the corresponding values are not updated
in the original dataframe.
- >>> df = pd.DataFrame({'A': [1, 2, 3],
- ... 'B': [400., 500., 600.]})
- >>> new_df = pd.DataFrame({'B': [4, np.nan, 6]})
+ >>> df = pd.DataFrame({"A": [1, 2, 3], "B": [400.0, 500.0, 600.0]})
+ >>> new_df = pd.DataFrame({"B": [4, np.nan, 6]})
>>> df.update(new_df)
>>> df
A B
@@ -8931,20 +8784,13 @@ def update(
1 2 500.0
2 3 6.0
"""
- if not PYPY and using_copy_on_write():
+ if not PYPY:
if sys.getrefcount(self) <= REF_COUNT:
warnings.warn(
_chained_assignment_method_msg,
ChainedAssignmentError,
stacklevel=2,
)
- elif not PYPY and not using_copy_on_write() and self._is_view_after_cow_rules():
- if sys.getrefcount(self) <= REF_COUNT:
- warnings.warn(
- _chained_assignment_warning_method_msg,
- FutureWarning,
- stacklevel=2,
- )
# TODO: Support other joins
if join != "left": # pragma: no cover
@@ -8955,11 +8801,22 @@ def update(
if not isinstance(other, DataFrame):
other = DataFrame(other)
- other = other.reindex(self.index)
+ if other.index.has_duplicates:
+ raise ValueError("Update not allowed with duplicate indexes on other.")
+
+ index_intersection = other.index.intersection(self.index)
+ if index_intersection.empty:
+ raise ValueError(
+ "Update not allowed when the index on `other` has no intersection "
+ "with this dataframe."
+ )
+
+ other = other.reindex(index_intersection)
+ this_data = self.loc[index_intersection]
for col in self.columns.intersection(other.columns):
- this = self[col]._values
- that = other[col]._values
+ this = this_data[col]
+ that = other[col]
if filter_func is not None:
mask = ~filter_func(this) | isna(that)
@@ -8979,7 +8836,7 @@ def update(
if mask.all():
continue
- self.loc[:, col] = self[col].where(mask, that)
+ self.loc[index_intersection, col] = this.where(mask, that)
# ----------------------------------------------------------------------
# Data reshaping
@@ -9034,8 +8891,8 @@ def update(
We can also choose to include NA in group keys or not by setting
`dropna` parameter, the default setting is `True`.
- >>> l = [[1, 2, 3], [1, None, 4], [2, 1, 3], [1, 2, 2]]
- >>> df = pd.DataFrame(l, columns=["a", "b", "c"])
+ >>> arr = [[1, 2, 3], [1, None, 4], [2, 1, 3], [1, 2, 2]]
+ >>> df = pd.DataFrame(arr, columns=["a", "b", "c"])
>>> df.groupby(by=["b"]).sum()
a c
@@ -9050,8 +8907,8 @@ def update(
2.0 2 5
NaN 1 4
- >>> l = [["a", 12, 12], [None, 12.3, 33.], ["b", 12.3, 123], ["a", 1, 1]]
- >>> df = pd.DataFrame(l, columns=["a", "b", "c"])
+ >>> arr = [["a", 12, 12], [None, 12.3, 33.], ["b", 12.3, 123], ["a", 1, 1]]
+ >>> df = pd.DataFrame(arr, columns=["a", "b", "c"])
>>> df.groupby(by="a").sum()
b c
@@ -9093,33 +8950,13 @@ def update(
def groupby(
self,
by=None,
- axis: Axis | lib.NoDefault = lib.no_default,
level: IndexLabel | None = None,
as_index: bool = True,
sort: bool = True,
group_keys: bool = True,
- observed: bool | lib.NoDefault = lib.no_default,
+ observed: bool = True,
dropna: bool = True,
) -> DataFrameGroupBy:
- if axis is not lib.no_default:
- axis = self._get_axis_number(axis)
- if axis == 1:
- warnings.warn(
- "DataFrame.groupby with axis=1 is deprecated. Do "
- "`frame.T.groupby(...)` without axis instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- warnings.warn(
- "The 'axis' keyword in DataFrame.groupby is deprecated and "
- "will be removed in a future version.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- axis = 0
-
from pandas.core.groupby.generic import DataFrameGroupBy
if level is None and by is None:
@@ -9128,7 +8965,6 @@ def groupby(
return DataFrameGroupBy(
obj=self,
keys=by,
- axis=axis,
level=level,
as_index=as_index,
sort=sort,
@@ -9137,9 +8973,7 @@ def groupby(
dropna=dropna,
)
- _shared_docs[
- "pivot"
- ] = """
+ _shared_docs["pivot"] = """
Return reshaped DataFrame organized by given index / column values.
Reshape data (produce a "pivot" table) based on column values. Uses
@@ -9224,11 +9058,11 @@ def groupby(
You could also assign a list of column names or a list of index names.
>>> df = pd.DataFrame({
- ... "lev1": [1, 1, 1, 2, 2, 2],
- ... "lev2": [1, 1, 2, 1, 1, 2],
- ... "lev3": [1, 2, 1, 2, 1, 2],
- ... "lev4": [1, 2, 3, 4, 5, 6],
- ... "values": [0, 1, 2, 3, 4, 5]})
+ ... "lev1": [1, 1, 1, 2, 2, 2],
+ ... "lev2": [1, 1, 2, 1, 1, 2],
+ ... "lev3": [1, 2, 1, 2, 1, 2],
+ ... "lev4": [1, 2, 3, 4, 5, 6],
+ ... "values": [0, 1, 2, 3, 4, 5]})
>>> df
lev1 lev2 lev3 lev4 values
0 1 1 1 1 0
@@ -9283,9 +9117,7 @@ def pivot(
return pivot(self, index=index, columns=columns, values=values)
- _shared_docs[
- "pivot_table"
- ] = """
+ _shared_docs["pivot_table"] = """
Create a spreadsheet-style pivot table as a DataFrame.
The levels in the pivot table will be stored in MultiIndex objects
@@ -9331,10 +9163,9 @@ def pivot(
If True: only show observed values for categorical groupers.
If False: show all values for categorical groupers.
- .. deprecated:: 2.2.0
+ .. versionchanged:: 3.0.0
- The default value of ``False`` is deprecated and will change to
- ``True`` in a future version of pandas.
+ The default value is now ``True``.
sort : bool, default True
Specifies if the result should be sorted.
@@ -9446,7 +9277,7 @@ def pivot_table(
margins: bool = False,
dropna: bool = True,
margins_name: Level = "All",
- observed: bool | lib.NoDefault = lib.no_default,
+ observed: bool = True,
sort: bool = True,
) -> DataFrame:
from pandas.core.reshape.pivot import pivot_table
@@ -9470,7 +9301,7 @@ def stack(
level: IndexLabel = -1,
dropna: bool | lib.NoDefault = lib.no_default,
sort: bool | lib.NoDefault = lib.no_default,
- future_stack: bool = False,
+ future_stack: bool = True,
):
"""
Stack the prescribed level(s) from columns to index.
@@ -9480,10 +9311,9 @@ def stack(
DataFrame. The new inner-most levels are created by pivoting the
columns of the current dataframe:
- - if the columns have a single level, the output is a Series;
- - if the columns have multiple levels, the new index
- level(s) is (are) taken from the prescribed level(s) and
- the output is a DataFrame.
+ - if the columns have a single level, the output is a Series;
+ - if the columns have multiple levels, the new index level(s) is (are)
+ taken from the prescribed level(s) and the output is a DataFrame.
Parameters
----------
@@ -9499,7 +9329,7 @@ def stack(
section.
sort : bool, default True
Whether to sort the levels of the resulting MultiIndex.
- future_stack : bool, default False
+ future_stack : bool, default True
Whether to use the new implementation that will replace the current
implementation in pandas 3.0. When True, dropna and sort have no impact
on the result and must remain unspecified. See :ref:`pandas 2.1.0 Release
@@ -9533,9 +9363,9 @@ def stack(
--------
**Single level columns**
- >>> df_single_level_cols = pd.DataFrame([[0, 1], [2, 3]],
- ... index=['cat', 'dog'],
- ... columns=['weight', 'height'])
+ >>> df_single_level_cols = pd.DataFrame(
+ ... [[0, 1], [2, 3]], index=["cat", "dog"], columns=["weight", "height"]
+ ... )
Stacking a dataframe with a single level column axis returns a Series:
@@ -9543,7 +9373,7 @@ def stack(
weight height
cat 0 1
dog 2 3
- >>> df_single_level_cols.stack(future_stack=True)
+ >>> df_single_level_cols.stack()
cat weight 0
height 1
dog weight 2
@@ -9552,11 +9382,12 @@ def stack(
**Multi level columns: simple case**
- >>> multicol1 = pd.MultiIndex.from_tuples([('weight', 'kg'),
- ... ('weight', 'pounds')])
- >>> df_multi_level_cols1 = pd.DataFrame([[1, 2], [2, 4]],
- ... index=['cat', 'dog'],
- ... columns=multicol1)
+ >>> multicol1 = pd.MultiIndex.from_tuples(
+ ... [("weight", "kg"), ("weight", "pounds")]
+ ... )
+ >>> df_multi_level_cols1 = pd.DataFrame(
+ ... [[1, 2], [2, 4]], index=["cat", "dog"], columns=multicol1
+ ... )
Stacking a dataframe with a multi-level column axis:
@@ -9565,7 +9396,7 @@ def stack(
kg pounds
cat 1 2
dog 2 4
- >>> df_multi_level_cols1.stack(future_stack=True)
+ >>> df_multi_level_cols1.stack()
weight
cat kg 1
pounds 2
@@ -9574,11 +9405,10 @@ def stack(
**Missing values**
- >>> multicol2 = pd.MultiIndex.from_tuples([('weight', 'kg'),
- ... ('height', 'm')])
- >>> df_multi_level_cols2 = pd.DataFrame([[1.0, 2.0], [3.0, 4.0]],
- ... index=['cat', 'dog'],
- ... columns=multicol2)
+ >>> multicol2 = pd.MultiIndex.from_tuples([("weight", "kg"), ("height", "m")])
+ >>> df_multi_level_cols2 = pd.DataFrame(
+ ... [[1.0, 2.0], [3.0, 4.0]], index=["cat", "dog"], columns=multicol2
+ ... )
It is common to have missing values when stacking a dataframe
with multi-level columns, as the stacked dataframe typically
@@ -9590,7 +9420,7 @@ def stack(
kg m
cat 1.0 2.0
dog 3.0 4.0
- >>> df_multi_level_cols2.stack(future_stack=True)
+ >>> df_multi_level_cols2.stack()
weight height
cat kg 1.0 NaN
m NaN 2.0
@@ -9601,13 +9431,13 @@ def stack(
The first parameter controls which level or levels are stacked:
- >>> df_multi_level_cols2.stack(0, future_stack=True)
+ >>> df_multi_level_cols2.stack(0)
kg m
cat weight 1.0 NaN
height NaN 2.0
dog weight 3.0 NaN
height NaN 4.0
- >>> df_multi_level_cols2.stack([0, 1], future_stack=True)
+ >>> df_multi_level_cols2.stack([0, 1])
cat weight kg 1.0
height m 2.0
dog weight kg 3.0
@@ -9620,19 +9450,14 @@ def stack(
stack_multiple,
)
- if (
- dropna is not lib.no_default
- or sort is not lib.no_default
- or self.columns.nlevels > 1
- ):
- warnings.warn(
- "The previous implementation of stack is deprecated and will be "
- "removed in a future version of pandas. See the What's New notes "
- "for pandas 2.1.0 for details. Specify future_stack=True to adopt "
- "the new implementation and silence this warning.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
+ warnings.warn(
+ "The previous implementation of stack is deprecated and will be "
+ "removed in a future version of pandas. See the What's New notes "
+ "for pandas 2.1.0 for details. Do not specify the future_stack "
+ "argument to adopt the new implementation and silence this warning.",
+ FutureWarning,
+ stacklevel=find_stack_level(),
+ )
if dropna is lib.no_default:
dropna = True
@@ -9648,14 +9473,14 @@ def stack(
if dropna is not lib.no_default:
raise ValueError(
- "dropna must be unspecified with future_stack=True as the new "
+ "dropna must be unspecified as the new "
"implementation does not introduce rows of NA values. This "
"argument will be removed in a future version of pandas."
)
if sort is not lib.no_default:
raise ValueError(
- "Cannot specify sort with future_stack=True, this argument will be "
+ "Cannot specify sort, this argument will be "
"removed in a future version of pandas. Sort the result using "
".sort_index instead."
)
@@ -9732,9 +9557,13 @@ def explode(
Examples
--------
- >>> df = pd.DataFrame({'A': [[0, 1, 2], 'foo', [], [3, 4]],
- ... 'B': 1,
- ... 'C': [['a', 'b', 'c'], np.nan, [], ['d', 'e']]})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "A": [[0, 1, 2], "foo", [], [3, 4]],
+ ... "B": 1,
+ ... "C": [["a", "b", "c"], np.nan, [], ["d", "e"]],
+ ... }
+ ... )
>>> df
A B C
0 [0, 1, 2] 1 [a, b, c]
@@ -9744,7 +9573,7 @@ def explode(
Single-column explode.
- >>> df.explode('A')
+ >>> df.explode("A")
A B C
0 0 1 [a, b, c]
0 1 1 [a, b, c]
@@ -9756,7 +9585,7 @@ def explode(
Multi-column explode.
- >>> df.explode(list('AC'))
+ >>> df.explode(list("AC"))
A B C
0 0 1 a
0 1 1 b
@@ -9801,11 +9630,13 @@ def explode(
result.index = default_index(len(result))
else:
result.index = self.index.take(result.index)
- result = result.reindex(columns=self.columns, copy=False)
+ result = result.reindex(columns=self.columns)
return result.__finalize__(self, method="explode")
- def unstack(self, level: IndexLabel = -1, fill_value=None, sort: bool = True):
+ def unstack(
+ self, level: IndexLabel = -1, fill_value=None, sort: bool = True
+ ) -> DataFrame | Series:
"""
Pivot a level of the (necessarily hierarchical) index labels.
@@ -9840,8 +9671,9 @@ def unstack(self, level: IndexLabel = -1, fill_value=None, sort: bool = True):
Examples
--------
- >>> index = pd.MultiIndex.from_tuples([('one', 'a'), ('one', 'b'),
- ... ('two', 'a'), ('two', 'b')])
+ >>> index = pd.MultiIndex.from_tuples(
+ ... [("one", "a"), ("one", "b"), ("two", "a"), ("two", "b")]
+ ... )
>>> s = pd.Series(np.arange(1.0, 5.0), index=index)
>>> s
one a 1.0
@@ -9874,7 +9706,6 @@ def unstack(self, level: IndexLabel = -1, fill_value=None, sort: bool = True):
return result.__finalize__(self, method="unstack")
- @Appender(_shared_docs["melt"] % {"caller": "df.melt(", "other": "melt"})
def melt(
self,
id_vars=None,
@@ -9884,6 +9715,127 @@ def melt(
col_level: Level | None = None,
ignore_index: bool = True,
) -> DataFrame:
+ """
+ Unpivot DataFrame from wide to long format, optionally leaving identifiers set.
+
+ This function is useful to massage a DataFrame into a format where one
+ or more columns are identifier variables (`id_vars`), while all other
+ columns, considered measured variables (`value_vars`), are "unpivoted" to
+ the row axis, leaving just two non-identifier columns, 'variable' and
+ 'value'.
+
+ Parameters
+ ----------
+ id_vars : scalar, tuple, list, or ndarray, optional
+ Column(s) to use as identifier variables.
+ value_vars : scalar, tuple, list, or ndarray, optional
+ Column(s) to unpivot. If not specified, uses all columns that
+ are not set as `id_vars`.
+ var_name : scalar, default None
+ Name to use for the 'variable' column. If None it uses
+ ``frame.columns.name`` or 'variable'.
+ value_name : scalar, default 'value'
+ Name to use for the 'value' column, can't be an existing column label.
+ col_level : scalar, optional
+ If columns are a MultiIndex then use this level to melt.
+ ignore_index : bool, default True
+ If True, original index is ignored. If False, original index is retained.
+ Index labels will be repeated as necessary.
+
+ Returns
+ -------
+ DataFrame
+ Unpivoted DataFrame.
+
+ See Also
+ --------
+ melt : Identical method.
+ pivot_table : Create a spreadsheet-style pivot table as a DataFrame.
+ DataFrame.pivot : Return reshaped DataFrame organized
+ by given index / column values.
+ DataFrame.explode : Explode a DataFrame from list-like
+ columns to long format.
+
+ Notes
+ -----
+ Reference :ref:`the user guide ` for more examples.
+
+ Examples
+ --------
+ >>> df = pd.DataFrame(
+ ... {
+ ... "A": {0: "a", 1: "b", 2: "c"},
+ ... "B": {0: 1, 1: 3, 2: 5},
+ ... "C": {0: 2, 1: 4, 2: 6},
+ ... }
+ ... )
+ >>> df
+ A B C
+ 0 a 1 2
+ 1 b 3 4
+ 2 c 5 6
+
+ >>> df.melt(id_vars=["A"], value_vars=["B"])
+ A variable value
+ 0 a B 1
+ 1 b B 3
+ 2 c B 5
+
+ >>> df.melt(id_vars=["A"], value_vars=["B", "C"])
+ A variable value
+ 0 a B 1
+ 1 b B 3
+ 2 c B 5
+ 3 a C 2
+ 4 b C 4
+ 5 c C 6
+
+ The names of 'variable' and 'value' columns can be customized:
+
+ >>> df.melt(
+ ... id_vars=["A"],
+ ... value_vars=["B"],
+ ... var_name="myVarname",
+ ... value_name="myValname",
+ ... )
+ A myVarname myValname
+ 0 a B 1
+ 1 b B 3
+ 2 c B 5
+
+ Original index values can be kept around:
+
+ >>> df.melt(id_vars=["A"], value_vars=["B", "C"], ignore_index=False)
+ A variable value
+ 0 a B 1
+ 1 b B 3
+ 2 c B 5
+ 0 a C 2
+ 1 b C 4
+ 2 c C 6
+
+ If you have multi-index columns:
+
+ >>> df.columns = [list("ABC"), list("DEF")]
+ >>> df
+ A B C
+ D E F
+ 0 a 1 2
+ 1 b 3 4
+ 2 c 5 6
+
+ >>> df.melt(col_level=0, id_vars=["A"], value_vars=["B"])
+ A variable value
+ 0 a B 1
+ 1 b B 3
+ 2 c B 5
+
+ >>> df.melt(id_vars=[("A", "D")], value_vars=[("B", "E")])
+ (A, D) variable_0 variable_1 value
+ 0 a B E 1
+ 1 b B E 3
+ 2 c B E 5
+ """
return melt(
self,
id_vars=id_vars,
@@ -10024,11 +9976,11 @@ def _gotitem(
--------
DataFrame.apply : Perform any type of operations.
DataFrame.transform : Perform transformation type operations.
- pandas.DataFrame.groupby : Perform operations over groups.
- pandas.DataFrame.resample : Perform operations over resampled bins.
- pandas.DataFrame.rolling : Perform operations over rolling window.
- pandas.DataFrame.expanding : Perform operations over expanding window.
- pandas.core.window.ewm.ExponentialMovingWindow : Perform operation over exponential
+ DataFrame.groupby : Perform operations over groups.
+ DataFrame.resample : Perform operations over resampled bins.
+ DataFrame.rolling : Perform operations over rolling window.
+ DataFrame.expanding : Perform operations over expanding window.
+ core.window.ewm.ExponentialMovingWindow : Perform operation over exponential
weighted window.
"""
)
@@ -10235,7 +10187,7 @@ def apply(
Examples
--------
- >>> df = pd.DataFrame([[4, 9]] * 3, columns=['A', 'B'])
+ >>> df = pd.DataFrame([[4, 9]] * 3, columns=["A", "B"])
>>> df
A B
0 4 9
@@ -10275,7 +10227,7 @@ def apply(
Passing ``result_type='expand'`` will expand list-like results
to columns of a Dataframe
- >>> df.apply(lambda x: [1, 2], axis=1, result_type='expand')
+ >>> df.apply(lambda x: [1, 2], axis=1, result_type="expand")
0 1
0 1 2
1 1 2
@@ -10285,7 +10237,7 @@ def apply(
``result_type='expand'``. The resulting column names
will be the Series index.
- >>> df.apply(lambda x: pd.Series([1, 2], index=['foo', 'bar']), axis=1)
+ >>> df.apply(lambda x: pd.Series([1, 2], index=["foo", "bar"]), axis=1)
foo bar
0 1 2
1 1 2
@@ -10296,7 +10248,7 @@ def apply(
and broadcast it along the axis. The resulting column names will
be the originals.
- >>> df.apply(lambda x: [1, 2], axis=1, result_type='broadcast')
+ >>> df.apply(lambda x: [1, 2], axis=1, result_type="broadcast")
A B
0 1 2
1 1 2
@@ -10369,7 +10321,7 @@ def map(
>>> df_copy = df.copy()
>>> df_copy.iloc[0, 0] = pd.NA
- >>> df_copy.map(lambda x: len(str(x)), na_action='ignore')
+ >>> df_copy.map(lambda x: len(str(x)), na_action="ignore")
0 1
0 NaN 4
1 5.0 5
@@ -10392,15 +10344,13 @@ def map(
But it's better to avoid map in that case.
- >>> df ** 2
+ >>> df**2
0 1
0 1.000000 4.494400
1 11.262736 20.857489
"""
if na_action not in {"ignore", None}:
- raise ValueError(
- f"na_action must be 'ignore' or None. Got {repr(na_action)}"
- )
+ raise ValueError(f"na_action must be 'ignore' or None. Got {na_action!r}")
if self.empty:
return self.copy()
@@ -10412,60 +10362,6 @@ def infer(x):
return self.apply(infer).__finalize__(self, "map")
- def applymap(
- self, func: PythonFuncType, na_action: NaAction | None = None, **kwargs
- ) -> DataFrame:
- """
- Apply a function to a Dataframe elementwise.
-
- .. deprecated:: 2.1.0
-
- DataFrame.applymap has been deprecated. Use DataFrame.map instead.
-
- This method applies a function that accepts and returns a scalar
- to every element of a DataFrame.
-
- Parameters
- ----------
- func : callable
- Python function, returns a single value from a single value.
- na_action : {None, 'ignore'}, default None
- If 'ignore', propagate NaN values, without passing them to func.
- **kwargs
- Additional keyword arguments to pass as keywords arguments to
- `func`.
-
- Returns
- -------
- DataFrame
- Transformed DataFrame.
-
- See Also
- --------
- DataFrame.apply : Apply a function along input axis of DataFrame.
- DataFrame.map : Apply a function along input axis of DataFrame.
- DataFrame.replace: Replace values given in `to_replace` with `value`.
-
- Examples
- --------
- >>> df = pd.DataFrame([[1, 2.12], [3.356, 4.567]])
- >>> df
- 0 1
- 0 1.000 2.120
- 1 3.356 4.567
-
- >>> df.map(lambda x: len(str(x)))
- 0 1
- 0 3 4
- 1 5 5
- """
- warnings.warn(
- "DataFrame.applymap has been deprecated. Use DataFrame.map instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self.map(func, na_action=na_action, **kwargs)
-
# ----------------------------------------------------------------------
# Merging / joining methods
@@ -10496,9 +10392,7 @@ def _append(
row_df = other.to_frame().T
# infer_objects is needed for
# test_append_empty_frame_to_series_with_dateutil_tz
- other = row_df.infer_objects(copy=False).rename_axis(
- index.names, copy=False
- )
+ other = row_df.infer_objects().rename_axis(index.names)
elif isinstance(other, list):
if not other:
pass
@@ -10597,8 +10491,12 @@ def join(
Examples
--------
- >>> df = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3', 'K4', 'K5'],
- ... 'A': ['A0', 'A1', 'A2', 'A3', 'A4', 'A5']})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "key": ["K0", "K1", "K2", "K3", "K4", "K5"],
+ ... "A": ["A0", "A1", "A2", "A3", "A4", "A5"],
+ ... }
+ ... )
>>> df
key A
@@ -10609,8 +10507,7 @@ def join(
4 K4 A4
5 K5 A5
- >>> other = pd.DataFrame({'key': ['K0', 'K1', 'K2'],
- ... 'B': ['B0', 'B1', 'B2']})
+ >>> other = pd.DataFrame({"key": ["K0", "K1", "K2"], "B": ["B0", "B1", "B2"]})
>>> other
key B
@@ -10620,7 +10517,7 @@ def join(
Join DataFrames using their indexes.
- >>> df.join(other, lsuffix='_caller', rsuffix='_other')
+ >>> df.join(other, lsuffix="_caller", rsuffix="_other")
key_caller A key_other B
0 K0 A0 K0 B0
1 K1 A1 K1 B1
@@ -10633,7 +10530,7 @@ def join(
the index in both `df` and `other`. The joined DataFrame will have
key as its index.
- >>> df.set_index('key').join(other.set_index('key'))
+ >>> df.set_index("key").join(other.set_index("key"))
A B
key
K0 A0 B0
@@ -10648,7 +10545,7 @@ def join(
any column in `df`. This method preserves the original DataFrame's
index in the result.
- >>> df.join(other.set_index('key'), on='key')
+ >>> df.join(other.set_index("key"), on="key")
key A B
0 K0 A0 B0
1 K1 A1 B1
@@ -10659,8 +10556,12 @@ def join(
Using non-unique key values shows how they are matched.
- >>> df = pd.DataFrame({'key': ['K0', 'K1', 'K1', 'K3', 'K0', 'K1'],
- ... 'A': ['A0', 'A1', 'A2', 'A3', 'A4', 'A5']})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "key": ["K0", "K1", "K1", "K3", "K0", "K1"],
+ ... "A": ["A0", "A1", "A2", "A3", "A4", "A5"],
+ ... }
+ ... )
>>> df
key A
@@ -10671,7 +10572,7 @@ def join(
4 K0 A4
5 K1 A5
- >>> df.join(other.set_index('key'), on='key', validate='m:1')
+ >>> df.join(other.set_index("key"), on="key", validate="m:1")
key A B
0 K0 A0 B0
1 K1 A1 B1
@@ -10735,7 +10636,7 @@ def join(
res = concat(
frames, axis=1, join="outer", verify_integrity=True, sort=sort
)
- return res.reindex(self.index, copy=False)
+ return res.reindex(self.index)
else:
return concat(
frames, axis=1, join=how, verify_integrity=True, sort=sort
@@ -10768,10 +10669,12 @@ def merge(
right_index: bool = False,
sort: bool = False,
suffixes: Suffixes = ("_x", "_y"),
- copy: bool | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
indicator: str | bool = False,
validate: MergeValidate | None = None,
) -> DataFrame:
+ self._check_copy_deprecation(copy)
+
from pandas.core.reshape.merge import merge
return merge(
@@ -10785,7 +10688,6 @@ def merge(
right_index=right_index,
sort=sort,
suffixes=suffixes,
- copy=copy,
indicator=indicator,
validate=validate,
)
@@ -10825,10 +10727,18 @@ def round(
numpy.around : Round a numpy array to the given number of decimals.
Series.round : Round a Series to the given number of decimals.
+ Notes
+ -----
+ For values exactly halfway between rounded decimal values, pandas rounds
+ to the nearest even value (e.g. -0.5 and 0.5 round to 0.0, 1.5 and 2.5
+ round to 2.0, etc.).
+
Examples
--------
- >>> df = pd.DataFrame([(.21, .32), (.01, .67), (.66, .03), (.21, .18)],
- ... columns=['dogs', 'cats'])
+ >>> df = pd.DataFrame(
+ ... [(0.21, 0.32), (0.01, 0.67), (0.66, 0.03), (0.21, 0.18)],
+ ... columns=["dogs", "cats"],
+ ... )
>>> df
dogs cats
0 0.21 0.32
@@ -10850,7 +10760,7 @@ def round(
specified with the column names as key and the number of decimal
places as value
- >>> df.round({'dogs': 1, 'cats': 0})
+ >>> df.round({"dogs": 1, "cats": 0})
dogs cats
0 0.2 0.0
1 0.0 1.0
@@ -10861,7 +10771,7 @@ def round(
specified with the column names as index and the number of
decimal places as value
- >>> decimals = pd.Series([0, 1], index=['cats', 'dogs'])
+ >>> decimals = pd.Series([0, 1], index=["cats", "dogs"])
>>> df.round(decimals)
dogs cats
0 0.2 0.0
@@ -10871,7 +10781,7 @@ def round(
"""
from pandas.core.reshape.concat import concat
- def _dict_round(df: DataFrame, decimals):
+ def _dict_round(df: DataFrame, decimals) -> Iterator[Series]:
for col, vals in df.items():
try:
yield _series_round(vals, decimals[col])
@@ -10899,7 +10809,6 @@ def _series_round(ser: Series, decimals: int) -> Series:
# type "Union[int, integer[Any]]"; expected "int"
new_mgr = self._mgr.round(
decimals=decimals, # type: ignore[arg-type]
- using_cow=using_copy_on_write(),
)
return self._constructor_from_mgr(new_mgr, axes=new_mgr.axes).__finalize__(
self, method="round"
@@ -10974,15 +10883,18 @@ def corr(
>>> def histogram_intersection(a, b):
... v = np.minimum(a, b).sum().round(decimals=1)
... return v
- >>> df = pd.DataFrame([(.2, .3), (.0, .6), (.6, .0), (.2, .1)],
- ... columns=['dogs', 'cats'])
+ >>> df = pd.DataFrame(
+ ... [(0.2, 0.3), (0.0, 0.6), (0.6, 0.0), (0.2, 0.1)],
+ ... columns=["dogs", "cats"],
+ ... )
>>> df.corr(method=histogram_intersection)
dogs cats
dogs 1.0 0.3
cats 0.3 1.0
- >>> df = pd.DataFrame([(1, 1), (2, np.nan), (np.nan, 3), (4, 4)],
- ... columns=['dogs', 'cats'])
+ >>> df = pd.DataFrame(
+ ... [(1, 1), (2, np.nan), (np.nan, 3), (4, 4)], columns=["dogs", "cats"]
+ ... )
>>> df.corr(min_periods=3)
dogs cats
dogs 1.0 NaN
@@ -11108,16 +11020,18 @@ def cov(
Examples
--------
- >>> df = pd.DataFrame([(1, 2), (0, 3), (2, 0), (1, 1)],
- ... columns=['dogs', 'cats'])
+ >>> df = pd.DataFrame(
+ ... [(1, 2), (0, 3), (2, 0), (1, 1)], columns=["dogs", "cats"]
+ ... )
>>> df.cov()
dogs cats
dogs 0.666667 -1.000000
cats -1.000000 1.666667
>>> np.random.seed(42)
- >>> df = pd.DataFrame(np.random.randn(1000, 5),
- ... columns=['a', 'b', 'c', 'd', 'e'])
+ >>> df = pd.DataFrame(
+ ... np.random.randn(1000, 5), columns=["a", "b", "c", "d", "e"]
+ ... )
>>> df.cov()
a b c d e
a 0.998438 -0.020161 0.059277 -0.008943 0.014144
@@ -11133,10 +11047,9 @@ def cov(
each column pair in order to have a valid result:
>>> np.random.seed(42)
- >>> df = pd.DataFrame(np.random.randn(20, 3),
- ... columns=['a', 'b', 'c'])
- >>> df.loc[df.index[:5], 'a'] = np.nan
- >>> df.loc[df.index[5:10], 'b'] = np.nan
+ >>> df = pd.DataFrame(np.random.randn(20, 3), columns=["a", "b", "c"])
+ >>> df.loc[df.index[:5], "a"] = np.nan
+ >>> df.loc[df.index[5:10], "b"] = np.nan
>>> df.cov(min_periods=12)
a b c
a 0.316741 NaN -0.150812
@@ -11216,8 +11129,12 @@ def corrwith(
--------
>>> index = ["a", "b", "c", "d", "e"]
>>> columns = ["one", "two", "three", "four"]
- >>> df1 = pd.DataFrame(np.arange(20).reshape(5, 4), index=index, columns=columns)
- >>> df2 = pd.DataFrame(np.arange(16).reshape(4, 4), index=index[:4], columns=columns)
+ >>> df1 = pd.DataFrame(
+ ... np.arange(20).reshape(5, 4), index=index, columns=columns
+ ... )
+ >>> df2 = pd.DataFrame(
+ ... np.arange(16).reshape(4, 4), index=index[:4], columns=columns
+ ... )
>>> df1.corrwith(df2)
one 1.0
two 1.0
@@ -11232,7 +11149,7 @@ def corrwith(
d 1.0
e NaN
dtype: float64
- """ # noqa: E501
+ """
axis = self._get_axis_number(axis)
this = self._get_numeric_data() if numeric_only else self
@@ -11241,7 +11158,7 @@ def corrwith(
if numeric_only:
other = other._get_numeric_data()
- left, right = this.align(other, join="inner", copy=False)
+ left, right = this.align(other, join="inner")
if axis == 1:
left = left.T
@@ -11300,7 +11217,7 @@ def c(x):
# ----------------------------------------------------------------------
# ndarray-like stats methods
- def count(self, axis: Axis = 0, numeric_only: bool = False):
+ def count(self, axis: Axis = 0, numeric_only: bool = False) -> Series:
"""
Count non-NA cells for each column or row.
@@ -11332,10 +11249,13 @@ def count(self, axis: Axis = 0, numeric_only: bool = False):
--------
Constructing DataFrame from a dictionary:
- >>> df = pd.DataFrame({"Person":
- ... ["John", "Myla", "Lewis", "John", "Myla"],
- ... "Age": [24., np.nan, 21., 33, 26],
- ... "Single": [False, True, True, True, False]})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "Person": ["John", "Myla", "Lewis", "John", "Myla"],
+ ... "Age": [24.0, np.nan, 21.0, 33, 26],
+ ... "Single": [False, True, True, True, False],
+ ... }
+ ... )
>>> df
Person Age Single
0 John 24.0 False
@@ -11354,7 +11274,7 @@ def count(self, axis: Axis = 0, numeric_only: bool = False):
Counts for each **row**:
- >>> df.count(axis='columns')
+ >>> df.count(axis="columns")
0 3
1 2
2 3
@@ -11375,7 +11295,7 @@ def count(self, axis: Axis = 0, numeric_only: bool = False):
else:
result = notna(frame).sum(axis=axis)
- return result.astype("int64", copy=False).__finalize__(self, method="count")
+ return result.astype("int64").__finalize__(self, method="count")
def _reduce(
self,
@@ -11402,9 +11322,7 @@ def func(values: np.ndarray):
def blk_func(values, axis: Axis = 1):
if isinstance(values, ExtensionArray):
- if not is_1d_only_ea_dtype(values.dtype) and not isinstance(
- self._mgr, ArrayManager
- ):
+ if not is_1d_only_ea_dtype(values.dtype):
return values._reduce(name, axis=1, skipna=skipna, **kwds)
has_keepdims = dtype_has_keepdims.get(values.dtype)
if has_keepdims is None:
@@ -11441,7 +11359,7 @@ def _get_data() -> DataFrame:
if axis is None:
dtype = find_common_type([arr.dtype for arr in df._mgr.arrays])
if isinstance(dtype, ExtensionDtype):
- df = df.astype(dtype, copy=False)
+ df = df.astype(dtype)
arr = concat_compat(list(df._iter_column_arrays()))
return arr._reduce(name, skipna=skipna, keepdims=False, **kwds)
return func(df.values)
@@ -11473,7 +11391,7 @@ def _get_data() -> DataFrame:
# be equivalent to transposing the original frame and aggregating
# with axis=0.
name = {"argmax": "idxmax", "argmin": "idxmin"}.get(name, name)
- df = df.astype(dtype, copy=False)
+ df = df.astype(dtype)
arr = concat_compat(list(df._iter_column_arrays()))
nrows, ncols = df.shape
row_index = np.tile(np.arange(nrows), ncols)
@@ -11545,25 +11463,85 @@ def _reduce_axis1(self, name: str, func, skipna: bool) -> Series:
res_ser = self._constructor_sliced(result, index=self.index, copy=False)
return res_ser
- @doc(make_doc("any", ndim=2))
# error: Signature of "any" incompatible with supertype "NDFrame"
- def any( # type: ignore[override]
+ @overload # type: ignore[override]
+ def any(
self,
*,
- axis: Axis | None = 0,
- bool_only: bool = False,
- skipna: bool = True,
+ axis: Axis = ...,
+ bool_only: bool = ...,
+ skipna: bool = ...,
**kwargs,
- ) -> Series | bool:
- result = self._logical_func(
- "any", nanops.nanany, axis, bool_only, skipna, **kwargs
- )
- if isinstance(result, Series):
- result = result.__finalize__(self, method="any")
- return result
+ ) -> Series: ...
- @doc(make_doc("all", ndim=2))
- def all(
+ @overload
+ def any(
+ self,
+ *,
+ axis: None,
+ bool_only: bool = ...,
+ skipna: bool = ...,
+ **kwargs,
+ ) -> bool: ...
+
+ @overload
+ def any(
+ self,
+ *,
+ axis: Axis | None,
+ bool_only: bool = ...,
+ skipna: bool = ...,
+ **kwargs,
+ ) -> Series | bool: ...
+
+ @doc(make_doc("any", ndim=2))
+ def any(
+ self,
+ *,
+ axis: Axis | None = 0,
+ bool_only: bool = False,
+ skipna: bool = True,
+ **kwargs,
+ ) -> Series | bool:
+ result = self._logical_func(
+ "any", nanops.nanany, axis, bool_only, skipna, **kwargs
+ )
+ if isinstance(result, Series):
+ result = result.__finalize__(self, method="any")
+ return result
+
+ @overload
+ def all(
+ self,
+ *,
+ axis: Axis = ...,
+ bool_only: bool = ...,
+ skipna: bool = ...,
+ **kwargs,
+ ) -> Series: ...
+
+ @overload
+ def all(
+ self,
+ *,
+ axis: None,
+ bool_only: bool = ...,
+ skipna: bool = ...,
+ **kwargs,
+ ) -> bool: ...
+
+ @overload
+ def all(
+ self,
+ *,
+ axis: Axis | None,
+ bool_only: bool = ...,
+ skipna: bool = ...,
+ **kwargs,
+ ) -> Series | bool: ...
+
+ @doc(make_doc("all", ndim=2))
+ def all(
self,
axis: Axis | None = 0,
bool_only: bool = False,
@@ -11577,6 +11555,37 @@ def all(
result = result.__finalize__(self, method="all")
return result
+ # error: Signature of "min" incompatible with supertype "NDFrame"
+ @overload # type: ignore[override]
+ def min(
+ self,
+ *,
+ axis: Axis = ...,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series: ...
+
+ @overload
+ def min(
+ self,
+ *,
+ axis: None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Any: ...
+
+ @overload
+ def min(
+ self,
+ *,
+ axis: Axis | None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series | Any: ...
+
@doc(make_doc("min", ndim=2))
def min(
self,
@@ -11584,12 +11593,45 @@ def min(
skipna: bool = True,
numeric_only: bool = False,
**kwargs,
- ):
- result = super().min(axis, skipna, numeric_only, **kwargs)
+ ) -> Series | Any:
+ result = super().min(
+ axis=axis, skipna=skipna, numeric_only=numeric_only, **kwargs
+ )
if isinstance(result, Series):
result = result.__finalize__(self, method="min")
return result
+ # error: Signature of "max" incompatible with supertype "NDFrame"
+ @overload # type: ignore[override]
+ def max(
+ self,
+ *,
+ axis: Axis = ...,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series: ...
+
+ @overload
+ def max(
+ self,
+ *,
+ axis: None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Any: ...
+
+ @overload
+ def max(
+ self,
+ *,
+ axis: Axis | None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series | Any: ...
+
@doc(make_doc("max", ndim=2))
def max(
self,
@@ -11597,8 +11639,10 @@ def max(
skipna: bool = True,
numeric_only: bool = False,
**kwargs,
- ):
- result = super().max(axis, skipna, numeric_only, **kwargs)
+ ) -> Series | Any:
+ result = super().max(
+ axis=axis, skipna=skipna, numeric_only=numeric_only, **kwargs
+ )
if isinstance(result, Series):
result = result.__finalize__(self, method="max")
return result
@@ -11611,9 +11655,17 @@ def sum(
numeric_only: bool = False,
min_count: int = 0,
**kwargs,
- ):
- result = super().sum(axis, skipna, numeric_only, min_count, **kwargs)
- return result.__finalize__(self, method="sum")
+ ) -> Series:
+ result = super().sum(
+ axis=axis,
+ skipna=skipna,
+ numeric_only=numeric_only,
+ min_count=min_count,
+ **kwargs,
+ )
+ if isinstance(result, Series):
+ result = result.__finalize__(self, method="sum")
+ return result
@doc(make_doc("prod", ndim=2))
def prod(
@@ -11623,9 +11675,48 @@ def prod(
numeric_only: bool = False,
min_count: int = 0,
**kwargs,
- ):
- result = super().prod(axis, skipna, numeric_only, min_count, **kwargs)
- return result.__finalize__(self, method="prod")
+ ) -> Series:
+ result = super().prod(
+ axis=axis,
+ skipna=skipna,
+ numeric_only=numeric_only,
+ min_count=min_count,
+ **kwargs,
+ )
+ if isinstance(result, Series):
+ result = result.__finalize__(self, method="prod")
+ return result
+
+ # error: Signature of "mean" incompatible with supertype "NDFrame"
+ @overload # type: ignore[override]
+ def mean(
+ self,
+ *,
+ axis: Axis = ...,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series: ...
+
+ @overload
+ def mean(
+ self,
+ *,
+ axis: None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Any: ...
+
+ @overload
+ def mean(
+ self,
+ *,
+ axis: Axis | None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series | Any: ...
@doc(make_doc("mean", ndim=2))
def mean(
@@ -11634,12 +11725,45 @@ def mean(
skipna: bool = True,
numeric_only: bool = False,
**kwargs,
- ):
- result = super().mean(axis, skipna, numeric_only, **kwargs)
+ ) -> Series | Any:
+ result = super().mean(
+ axis=axis, skipna=skipna, numeric_only=numeric_only, **kwargs
+ )
if isinstance(result, Series):
result = result.__finalize__(self, method="mean")
return result
+ # error: Signature of "median" incompatible with supertype "NDFrame"
+ @overload # type: ignore[override]
+ def median(
+ self,
+ *,
+ axis: Axis = ...,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series: ...
+
+ @overload
+ def median(
+ self,
+ *,
+ axis: None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Any: ...
+
+ @overload
+ def median(
+ self,
+ *,
+ axis: Axis | None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series | Any: ...
+
@doc(make_doc("median", ndim=2))
def median(
self,
@@ -11647,12 +11771,48 @@ def median(
skipna: bool = True,
numeric_only: bool = False,
**kwargs,
- ):
- result = super().median(axis, skipna, numeric_only, **kwargs)
+ ) -> Series | Any:
+ result = super().median(
+ axis=axis, skipna=skipna, numeric_only=numeric_only, **kwargs
+ )
if isinstance(result, Series):
result = result.__finalize__(self, method="median")
return result
+ # error: Signature of "sem" incompatible with supertype "NDFrame"
+ @overload # type: ignore[override]
+ def sem(
+ self,
+ *,
+ axis: Axis = ...,
+ skipna: bool = ...,
+ ddof: int = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series: ...
+
+ @overload
+ def sem(
+ self,
+ *,
+ axis: None,
+ skipna: bool = ...,
+ ddof: int = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Any: ...
+
+ @overload
+ def sem(
+ self,
+ *,
+ axis: Axis | None,
+ skipna: bool = ...,
+ ddof: int = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series | Any: ...
+
@doc(make_doc("sem", ndim=2))
def sem(
self,
@@ -11661,12 +11821,48 @@ def sem(
ddof: int = 1,
numeric_only: bool = False,
**kwargs,
- ):
- result = super().sem(axis, skipna, ddof, numeric_only, **kwargs)
+ ) -> Series | Any:
+ result = super().sem(
+ axis=axis, skipna=skipna, ddof=ddof, numeric_only=numeric_only, **kwargs
+ )
if isinstance(result, Series):
result = result.__finalize__(self, method="sem")
return result
+ # error: Signature of "var" incompatible with supertype "NDFrame"
+ @overload # type: ignore[override]
+ def var(
+ self,
+ *,
+ axis: Axis = ...,
+ skipna: bool = ...,
+ ddof: int = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series: ...
+
+ @overload
+ def var(
+ self,
+ *,
+ axis: None,
+ skipna: bool = ...,
+ ddof: int = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Any: ...
+
+ @overload
+ def var(
+ self,
+ *,
+ axis: Axis | None,
+ skipna: bool = ...,
+ ddof: int = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series | Any: ...
+
@doc(make_doc("var", ndim=2))
def var(
self,
@@ -11675,12 +11871,48 @@ def var(
ddof: int = 1,
numeric_only: bool = False,
**kwargs,
- ):
- result = super().var(axis, skipna, ddof, numeric_only, **kwargs)
+ ) -> Series | Any:
+ result = super().var(
+ axis=axis, skipna=skipna, ddof=ddof, numeric_only=numeric_only, **kwargs
+ )
if isinstance(result, Series):
result = result.__finalize__(self, method="var")
return result
+ # error: Signature of "std" incompatible with supertype "NDFrame"
+ @overload # type: ignore[override]
+ def std(
+ self,
+ *,
+ axis: Axis = ...,
+ skipna: bool = ...,
+ ddof: int = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series: ...
+
+ @overload
+ def std(
+ self,
+ *,
+ axis: None,
+ skipna: bool = ...,
+ ddof: int = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Any: ...
+
+ @overload
+ def std(
+ self,
+ *,
+ axis: Axis | None,
+ skipna: bool = ...,
+ ddof: int = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series | Any: ...
+
@doc(make_doc("std", ndim=2))
def std(
self,
@@ -11689,12 +11921,45 @@ def std(
ddof: int = 1,
numeric_only: bool = False,
**kwargs,
- ):
- result = super().std(axis, skipna, ddof, numeric_only, **kwargs)
+ ) -> Series | Any:
+ result = super().std(
+ axis=axis, skipna=skipna, ddof=ddof, numeric_only=numeric_only, **kwargs
+ )
if isinstance(result, Series):
result = result.__finalize__(self, method="std")
return result
+ # error: Signature of "skew" incompatible with supertype "NDFrame"
+ @overload # type: ignore[override]
+ def skew(
+ self,
+ *,
+ axis: Axis = ...,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series: ...
+
+ @overload
+ def skew(
+ self,
+ *,
+ axis: None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Any: ...
+
+ @overload
+ def skew(
+ self,
+ *,
+ axis: Axis | None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series | Any: ...
+
@doc(make_doc("skew", ndim=2))
def skew(
self,
@@ -11702,12 +11967,45 @@ def skew(
skipna: bool = True,
numeric_only: bool = False,
**kwargs,
- ):
- result = super().skew(axis, skipna, numeric_only, **kwargs)
+ ) -> Series | Any:
+ result = super().skew(
+ axis=axis, skipna=skipna, numeric_only=numeric_only, **kwargs
+ )
if isinstance(result, Series):
result = result.__finalize__(self, method="skew")
return result
+ # error: Signature of "kurt" incompatible with supertype "NDFrame"
+ @overload # type: ignore[override]
+ def kurt(
+ self,
+ *,
+ axis: Axis = ...,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series: ...
+
+ @overload
+ def kurt(
+ self,
+ *,
+ axis: None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Any: ...
+
+ @overload
+ def kurt(
+ self,
+ *,
+ axis: Axis | None,
+ skipna: bool = ...,
+ numeric_only: bool = ...,
+ **kwargs,
+ ) -> Series | Any: ...
+
@doc(make_doc("kurt", ndim=2))
def kurt(
self,
@@ -11715,29 +12013,32 @@ def kurt(
skipna: bool = True,
numeric_only: bool = False,
**kwargs,
- ):
- result = super().kurt(axis, skipna, numeric_only, **kwargs)
+ ) -> Series | Any:
+ result = super().kurt(
+ axis=axis, skipna=skipna, numeric_only=numeric_only, **kwargs
+ )
if isinstance(result, Series):
result = result.__finalize__(self, method="kurt")
return result
- kurtosis = kurt
+ # error: Incompatible types in assignment
+ kurtosis = kurt # type: ignore[assignment]
product = prod
@doc(make_doc("cummin", ndim=2))
- def cummin(self, axis: Axis | None = None, skipna: bool = True, *args, **kwargs):
+ def cummin(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
return NDFrame.cummin(self, axis, skipna, *args, **kwargs)
@doc(make_doc("cummax", ndim=2))
- def cummax(self, axis: Axis | None = None, skipna: bool = True, *args, **kwargs):
+ def cummax(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
return NDFrame.cummax(self, axis, skipna, *args, **kwargs)
@doc(make_doc("cumsum", ndim=2))
- def cumsum(self, axis: Axis | None = None, skipna: bool = True, *args, **kwargs):
+ def cumsum(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
return NDFrame.cumsum(self, axis, skipna, *args, **kwargs)
@doc(make_doc("cumprod", 2))
- def cumprod(self, axis: Axis | None = None, skipna: bool = True, *args, **kwargs):
+ def cumprod(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
return NDFrame.cumprod(self, axis, skipna, *args, **kwargs)
def nunique(self, axis: Axis = 0, dropna: bool = True) -> Series:
@@ -11758,6 +12059,7 @@ def nunique(self, axis: Axis = 0, dropna: bool = True) -> Series:
Returns
-------
Series
+ Series with counts of unique values per row or column, depending on `axis`.
See Also
--------
@@ -11766,7 +12068,7 @@ def nunique(self, axis: Axis = 0, dropna: bool = True) -> Series:
Examples
--------
- >>> df = pd.DataFrame({'A': [4, 5, 6], 'B': [4, 1, 1]})
+ >>> df = pd.DataFrame({"A": [4, 5, 6], "B": [4, 1, 1]})
>>> df.nunique()
A 3
B 2
@@ -11863,7 +12165,7 @@ def _get_agg_axis(self, axis_num: int) -> Index:
elif axis_num == 1:
return self.index
else:
- raise ValueError(f"Axis must be 0 or 1 (got {repr(axis_num)})")
+ raise ValueError(f"Axis must be 0 or 1 (got {axis_num!r})")
def mode(
self, axis: Axis = 0, numeric_only: bool = False, dropna: bool = True
@@ -11899,12 +12201,16 @@ def mode(
Examples
--------
- >>> df = pd.DataFrame([('bird', 2, 2),
- ... ('mammal', 4, np.nan),
- ... ('arthropod', 8, 0),
- ... ('bird', 2, np.nan)],
- ... index=('falcon', 'horse', 'spider', 'ostrich'),
- ... columns=('species', 'legs', 'wings'))
+ >>> df = pd.DataFrame(
+ ... [
+ ... ("bird", 2, 2),
+ ... ("mammal", 4, np.nan),
+ ... ("arthropod", 8, 0),
+ ... ("bird", 2, np.nan),
+ ... ],
+ ... index=("falcon", "horse", "spider", "ostrich"),
+ ... columns=("species", "legs", "wings"),
+ ... )
>>> df
species legs wings
falcon bird 2 2.0
@@ -11938,7 +12244,7 @@ def mode(
To compute the mode over columns and not rows, use the axis parameter:
- >>> df.mode(axis='columns', numeric_only=True)
+ >>> df.mode(axis="columns", numeric_only=True)
0 1
falcon 2.0 NaN
horse 4.0 NaN
@@ -11965,8 +12271,7 @@ def quantile(
numeric_only: bool = ...,
interpolation: QuantileInterpolation = ...,
method: Literal["single", "table"] = ...,
- ) -> Series:
- ...
+ ) -> Series: ...
@overload
def quantile(
@@ -11976,8 +12281,7 @@ def quantile(
numeric_only: bool = ...,
interpolation: QuantileInterpolation = ...,
method: Literal["single", "table"] = ...,
- ) -> Series | DataFrame:
- ...
+ ) -> Series | DataFrame: ...
@overload
def quantile(
@@ -11987,8 +12291,7 @@ def quantile(
numeric_only: bool = ...,
interpolation: QuantileInterpolation = ...,
method: Literal["single", "table"] = ...,
- ) -> Series | DataFrame:
- ...
+ ) -> Series | DataFrame: ...
def quantile(
self,
@@ -12045,24 +12348,25 @@ def quantile(
Examples
--------
- >>> df = pd.DataFrame(np.array([[1, 1], [2, 10], [3, 100], [4, 100]]),
- ... columns=['a', 'b'])
- >>> df.quantile(.1)
+ >>> df = pd.DataFrame(
+ ... np.array([[1, 1], [2, 10], [3, 100], [4, 100]]), columns=["a", "b"]
+ ... )
+ >>> df.quantile(0.1)
a 1.3
b 3.7
Name: 0.1, dtype: float64
- >>> df.quantile([.1, .5])
+ >>> df.quantile([0.1, 0.5])
a b
0.1 1.3 3.7
0.5 2.5 55.0
Specifying `method='table'` will compute the quantile over all columns.
- >>> df.quantile(.1, method="table", interpolation="nearest")
+ >>> df.quantile(0.1, method="table", interpolation="nearest")
a 1
b 1
Name: 0.1, dtype: int64
- >>> df.quantile([.1, .5], method="table", interpolation="nearest")
+ >>> df.quantile([0.1, 0.5], method="table", interpolation="nearest")
a b
0.1 1 1
0.5 3 100
@@ -12070,11 +12374,13 @@ def quantile(
Specifying `numeric_only=False` will also compute the quantile of
datetime and timedelta data.
- >>> df = pd.DataFrame({'A': [1, 2],
- ... 'B': [pd.Timestamp('2010'),
- ... pd.Timestamp('2011')],
- ... 'C': [pd.Timedelta('1 days'),
- ... pd.Timedelta('2 days')]})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "A": [1, 2],
+ ... "B": [pd.Timestamp("2010"), pd.Timestamp("2011")],
+ ... "C": [pd.Timedelta("1 days"), pd.Timedelta("2 days")],
+ ... }
+ ... )
>>> df.quantile(0.5, numeric_only=False)
A 1.5
B 2010-07-02 12:00:00
@@ -12170,7 +12476,7 @@ def to_timestamp(
freq: Frequency | None = None,
how: ToTimestampHow = "start",
axis: Axis = 0,
- copy: bool | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
) -> DataFrame:
"""
Cast to DatetimeIndex of timestamps, at *beginning* of period.
@@ -12184,7 +12490,7 @@ def to_timestamp(
vs. end.
axis : {0 or 'index', 1 or 'columns'}, default 0
The axis to convert (the index by default).
- copy : bool, default True
+ copy : bool, default False
If False then underlying input data is not copied.
.. note::
@@ -12199,6 +12505,8 @@ def to_timestamp(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+ .. deprecated:: 3.0.0
+
Returns
-------
DataFrame
@@ -12206,8 +12514,8 @@ def to_timestamp(
Examples
--------
- >>> idx = pd.PeriodIndex(['2023', '2024'], freq='Y')
- >>> d = {'col1': [1, 2], 'col2': [3, 4]}
+ >>> idx = pd.PeriodIndex(["2023", "2024"], freq="Y")
+ >>> d = {"col1": [1, 2], "col2": [3, 4]}
>>> df1 = pd.DataFrame(data=d, index=idx)
>>> df1
col1 col2
@@ -12227,7 +12535,7 @@ def to_timestamp(
Using `freq` which is the offset that the Timestamps will have
>>> df2 = pd.DataFrame(data=d, index=idx)
- >>> df2 = df2.to_timestamp(freq='M')
+ >>> df2 = df2.to_timestamp(freq="M")
>>> df2
col1 col2
2023-01-31 1 3
@@ -12235,7 +12543,8 @@ def to_timestamp(
>>> df2.index
DatetimeIndex(['2023-01-31', '2024-01-31'], dtype='datetime64[ns]', freq=None)
"""
- new_obj = self.copy(deep=copy and not using_copy_on_write())
+ self._check_copy_deprecation(copy)
+ new_obj = self.copy(deep=False)
axis_name = self._get_axis_name(axis)
old_ax = getattr(self, axis_name)
@@ -12248,7 +12557,10 @@ def to_timestamp(
return new_obj
def to_period(
- self, freq: Frequency | None = None, axis: Axis = 0, copy: bool | None = None
+ self,
+ freq: Frequency | None = None,
+ axis: Axis = 0,
+ copy: bool | lib.NoDefault = lib.no_default,
) -> DataFrame:
"""
Convert DataFrame from DatetimeIndex to PeriodIndex.
@@ -12262,7 +12574,7 @@ def to_period(
Frequency of the PeriodIndex.
axis : {0 or 'index', 1 or 'columns'}, default 0
The axis to convert (the index by default).
- copy : bool, default True
+ copy : bool, default False
If False then underlying input data is not copied.
.. note::
@@ -12277,6 +12589,8 @@ def to_period(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+ .. deprecated:: 3.0.0
+
Returns
-------
DataFrame
@@ -12304,7 +12618,8 @@ def to_period(
>>> idx.to_period("Y")
PeriodIndex(['2001', '2002', '2003'], dtype='period[Y-DEC]')
"""
- new_obj = self.copy(deep=copy and not using_copy_on_write())
+ self._check_copy_deprecation(copy)
+ new_obj = self.copy(deep=False)
axis_name = self._get_axis_name(axis)
old_ax = getattr(self, axis_name)
@@ -12344,8 +12659,9 @@ def isin(self, values: Series | DataFrame | Sequence | Mapping) -> DataFrame:
Examples
--------
- >>> df = pd.DataFrame({'num_legs': [2, 4], 'num_wings': [2, 0]},
- ... index=['falcon', 'dog'])
+ >>> df = pd.DataFrame(
+ ... {"num_legs": [2, 4], "num_wings": [2, 0]}, index=["falcon", "dog"]
+ ... )
>>> df
num_legs num_wings
falcon 2 2
@@ -12369,7 +12685,7 @@ def isin(self, values: Series | DataFrame | Sequence | Mapping) -> DataFrame:
When ``values`` is a dict, we can pass values to check for each
column separately:
- >>> df.isin({'num_wings': [0, 3]})
+ >>> df.isin({"num_wings": [0, 3]})
num_legs num_wings
falcon False False
dog False True
@@ -12378,8 +12694,9 @@ def isin(self, values: Series | DataFrame | Sequence | Mapping) -> DataFrame:
match. Note that 'falcon' does not match based on the number of legs
in other.
- >>> other = pd.DataFrame({'num_legs': [8, 3], 'num_wings': [0, 2]},
- ... index=['spider', 'falcon'])
+ >>> other = pd.DataFrame(
+ ... {"num_legs": [8, 3], "num_wings": [0, 2]}, index=["spider", "falcon"]
+ ... )
>>> df.isin(other)
num_legs num_wings
falcon False True
@@ -12525,11 +12842,9 @@ def _to_dict_of_blocks(self):
Internal ONLY - only works for BlockManager
"""
mgr = self._mgr
- # convert to BlockManager if needed -> this way support ArrayManager as well
- mgr = cast(BlockManager, mgr_to_mgr(mgr, "block"))
return {
k: self._constructor_from_mgr(v, axes=v.axes).__finalize__(self)
- for k, v, in mgr.to_dict().items()
+ for k, v in mgr.to_dict().items()
}
@property
@@ -12572,9 +12887,9 @@ def values(self) -> np.ndarray:
A DataFrame where all columns are the same type (e.g., int64) results
in an array of the same type.
- >>> df = pd.DataFrame({'age': [ 3, 29],
- ... 'height': [94, 170],
- ... 'weight': [31, 115]})
+ >>> df = pd.DataFrame(
+ ... {"age": [3, 29], "height": [94, 170], "weight": [31, 115]}
+ ... )
>>> df
age height weight
0 3 94 31
@@ -12592,10 +12907,14 @@ def values(self) -> np.ndarray:
results in an ndarray of the broadest type that accommodates these
mixed types (e.g., object).
- >>> df2 = pd.DataFrame([('parrot', 24.0, 'second'),
- ... ('lion', 80.5, 1),
- ... ('monkey', np.nan, None)],
- ... columns=('name', 'max_speed', 'rank'))
+ >>> df2 = pd.DataFrame(
+ ... [
+ ... ("parrot", 24.0, "second"),
+ ... ("lion", 80.5, 1),
+ ... ("monkey", np.nan, None),
+ ... ],
+ ... columns=("name", "max_speed", "rank"),
+ ... )
>>> df2.dtypes
name object
max_speed float64
@@ -12609,8 +12928,12 @@ def values(self) -> np.ndarray:
return self._mgr.as_array()
-def _from_nested_dict(data) -> collections.defaultdict:
- new_data: collections.defaultdict = collections.defaultdict(dict)
+def _from_nested_dict(
+ data: Mapping[HashableT, Mapping[HashableT2, T]],
+) -> collections.defaultdict[HashableT2, dict[HashableT, T]]:
+ new_data: collections.defaultdict[HashableT2, dict[HashableT, T]] = (
+ collections.defaultdict(dict)
+ )
for index, s in data.items():
for col, v in s.items():
new_data[col][index] = v
@@ -12623,7 +12946,7 @@ def _reindex_for_setitem(
# reindex if necessary
if value.index.equals(index) or not len(index):
- if using_copy_on_write() and isinstance(value, Series):
+ if isinstance(value, Series):
return value._values, value._references
return value._values.copy(), None
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
index 9b70c24aa67c6..858d2ba82a969 100644
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -5,7 +5,6 @@
from copy import deepcopy
import datetime as dt
from functools import partial
-import gc
from json import loads
import operator
import pickle
@@ -23,25 +22,18 @@
overload,
)
import warnings
-import weakref
import numpy as np
-from pandas._config import (
- config,
- using_copy_on_write,
- warn_copy_on_write,
-)
+from pandas._config import config
from pandas._libs import lib
from pandas._libs.lib import is_range_indexer
from pandas._libs.tslibs import (
Period,
- Tick,
Timestamp,
to_offset,
)
-from pandas._libs.tslibs.dtypes import freq_to_period_freqstr
from pandas._typing import (
AlignJoin,
AnyArrayLike,
@@ -50,6 +42,7 @@
Axis,
AxisInt,
CompressionOptions,
+ Concatenate,
DtypeArg,
DtypeBackend,
DtypeObj,
@@ -98,22 +91,14 @@
AbstractMethodError,
ChainedAssignmentError,
InvalidIndexError,
- SettingWithCopyError,
- SettingWithCopyWarning,
- _chained_assignment_method_msg,
- _chained_assignment_warning_method_msg,
- _check_cacher,
-)
-from pandas.util._decorators import (
- deprecate_nonkeyword_arguments,
- doc,
)
+from pandas.errors.cow import _chained_assignment_method_msg
+from pandas.util._decorators import doc
from pandas.util._exceptions import find_stack_level
from pandas.util._validators import (
check_dtype_backend,
validate_ascending,
validate_bool_kwarg,
- validate_fillna_kwargs,
validate_inclusive,
)
@@ -136,6 +121,7 @@
from pandas.core.dtypes.dtypes import (
DatetimeTZDtype,
ExtensionDtype,
+ PeriodDtype,
)
from pandas.core.dtypes.generic import (
ABCDataFrame,
@@ -173,15 +159,7 @@
default_index,
ensure_index,
)
-from pandas.core.internals import (
- ArrayManager,
- BlockManager,
- SingleArrayManager,
-)
-from pandas.core.internals.construction import (
- mgr_to_mgr,
- ndarray_to_mgr,
-)
+from pandas.core.internals import BlockManager
from pandas.core.methods.describe import describe_ndframe
from pandas.core.missing import (
clean_fill_method,
@@ -213,6 +191,7 @@
)
from pandas._libs.tslibs import BaseOffset
+ from pandas._typing import P
from pandas import (
DataFrame,
@@ -223,6 +202,8 @@
from pandas.core.indexers.objects import BaseIndexer
from pandas.core.resample import Resampler
+import textwrap
+
# goal is to be able to define the docs close to function, while still being
# able to share
_shared_docs = {**_shared_docs}
@@ -239,9 +220,6 @@
}
-bool_t = bool # Need alias because NDFrame has def bool:
-
-
class NDFrame(PandasObject, indexing.IndexingMixin):
"""
N-dimensional analogue of DataFrame. Store multi-dimensional in a
@@ -256,10 +234,8 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
_internal_names: list[str] = [
"_mgr",
- "_cacher",
"_item_cache",
"_cache",
- "_is_copy",
"_name",
"_metadata",
"_flags",
@@ -268,7 +244,6 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
_accessors: set[str] = set()
_hidden_attrs: frozenset[str] = frozenset([])
_metadata: list[str] = []
- _is_copy: weakref.ReferenceType[NDFrame] | str | None = None
_mgr: Manager
_attrs: dict[Hashable, Any]
_typ: str
@@ -277,9 +252,7 @@ class NDFrame(PandasObject, indexing.IndexingMixin):
# Constructors
def __init__(self, data: Manager) -> None:
- object.__setattr__(self, "_is_copy", None)
object.__setattr__(self, "_mgr", data)
- object.__setattr__(self, "_item_cache", {})
object.__setattr__(self, "_attrs", {})
object.__setattr__(self, "_flags", Flags(self, allows_duplicate_labels=True))
@@ -290,7 +263,7 @@ def _init_mgr(
mgr: Manager,
axes: dict[Literal["index", "columns"], Axes | None],
dtype: DtypeObj | None = None,
- copy: bool_t = False,
+ copy: bool = False,
) -> Manager:
"""passed a manager and a axes dict"""
for a, axe in axes.items():
@@ -315,28 +288,6 @@ def _init_mgr(
return mgr
@final
- def _as_manager(self, typ: str, copy: bool_t = True) -> Self:
- """
- Private helper function to create a DataFrame with specific manager.
-
- Parameters
- ----------
- typ : {"block", "array"}
- copy : bool, default True
- Only controls whether the conversion from Block->ArrayManager
- copies the 1D arrays (to ensure proper/contiguous memory layout).
-
- Returns
- -------
- DataFrame
- New DataFrame using specified manager type. Is not guaranteed
- to be a copy or not.
- """
- new_mgr: Manager
- new_mgr = mgr_to_mgr(self._mgr, typ=typ, copy=copy)
- # fastpath of passing a manager doesn't check the option/manager class
- return self._constructor_from_mgr(new_mgr, axes=new_mgr.axes).__finalize__(self)
-
@classmethod
def _from_mgr(cls, mgr: Manager, axes: list[Index]) -> Self:
"""
@@ -391,7 +342,7 @@ def attrs(self) -> dict[Hashable, Any]:
For DataFrame:
- >>> df = pd.DataFrame({'A': [1, 2], 'B': [3, 4]})
+ >>> df = pd.DataFrame({"A": [1, 2], "B": [3, 4]})
>>> df.attrs = {"A": [10, 20, 30]}
>>> df.attrs
{'A': [10, 20, 30]}
@@ -447,8 +398,8 @@ def flags(self) -> Flags:
def set_flags(
self,
*,
- copy: bool_t = False,
- allows_duplicate_labels: bool_t | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
+ allows_duplicate_labels: bool | None = None,
) -> Self:
"""
Return a new object with updated flags.
@@ -469,6 +420,8 @@ def set_flags(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+
+ .. deprecated:: 3.0.0
allows_duplicate_labels : bool, optional
Whether the returned object allows duplicate labels.
@@ -503,7 +456,8 @@ def set_flags(
>>> df2.flags.allows_duplicate_labels
False
"""
- df = self.copy(deep=copy and not using_copy_on_write())
+ self._check_copy_deprecation(copy)
+ df = self.copy(deep=False)
if allows_duplicate_labels is not None:
df.flags["allows_duplicate_labels"] = allows_duplicate_labels
return df
@@ -527,31 +481,15 @@ def _validate_dtype(cls, dtype) -> DtypeObj | None:
# ----------------------------------------------------------------------
# Construction
+ # error: Signature of "_constructor" incompatible with supertype "PandasObject"
@property
- def _constructor(self) -> Callable[..., Self]:
+ def _constructor(self) -> Callable[..., Self]: # type: ignore[override]
"""
Used when a manipulation result has the same dimensions as the
original.
"""
raise AbstractMethodError(self)
- # ----------------------------------------------------------------------
- # Internals
-
- @final
- @property
- def _data(self):
- # GH#33054 retained because some downstream packages uses this,
- # e.g. fastparquet
- # GH#33333
- warnings.warn(
- f"{type(self).__name__}._data is deprecated and will be removed in "
- "a future version. Use public APIs instead.",
- DeprecationWarning,
- stacklevel=find_stack_level(),
- )
- return self._mgr
-
# ----------------------------------------------------------------------
# Axis
_AXIS_ORDERS: list[Literal["index", "columns"]]
@@ -561,7 +499,9 @@ def _data(self):
_AXIS_LEN: int
@final
- def _construct_axes_dict(self, axes: Sequence[Axis] | None = None, **kwargs):
+ def _construct_axes_dict(
+ self, axes: Sequence[Axis] | None = None, **kwargs: AxisInt
+ ) -> dict:
"""Return an axes dictionary for myself."""
d = {a: self._get_axis(a) for a in (axes or self._AXIS_ORDERS)}
# error: Argument 1 to "update" of "MutableMapping" has incompatible type
@@ -574,8 +514,10 @@ def _construct_axes_dict(self, axes: Sequence[Axis] | None = None, **kwargs):
def _get_axis_number(cls, axis: Axis) -> AxisInt:
try:
return cls._AXIS_TO_AXIS_NUMBER[axis]
- except KeyError:
- raise ValueError(f"No axis named {axis} for object type {cls.__name__}")
+ except KeyError as err:
+ raise ValueError(
+ f"No axis named {axis} for object type {cls.__name__}"
+ ) from err
@final
@classmethod
@@ -644,7 +586,7 @@ def _get_index_resolvers(self) -> dict[Hashable, Series | MultiIndex]:
@final
def _get_cleaned_column_resolvers(self) -> dict[Hashable, Series]:
"""
- Return the special character free column resolvers of a dataframe.
+ Return the special character free column resolvers of a DataFrame.
Column names with special characters are 'cleaned up' so that they can
be referred to by backtick quoting.
@@ -658,7 +600,7 @@ def _get_cleaned_column_resolvers(self) -> dict[Hashable, Series]:
return {
clean_column_name(k): Series(
- v, copy=False, index=self.index, name=k
+ v, copy=False, index=self.index, name=k, dtype=self.dtypes[k]
).__finalize__(self)
for k, v in zip(self.columns, self._iter_column_arrays())
if not isinstance(k, int)
@@ -669,14 +611,6 @@ def _get_cleaned_column_resolvers(self) -> dict[Hashable, Series]:
def _info_axis(self) -> Index:
return getattr(self, self._info_axis_name)
- def _is_view_after_cow_rules(self):
- # Only to be used in cases of chained assignment checks, this is a
- # simplified check that assumes that either the whole object is a view
- # or a copy
- if len(self._mgr.blocks) == 0: # type: ignore[union-attr]
- return False
- return self._mgr.blocks[0].refs.has_reference() # type: ignore[union-attr]
-
@property
def shape(self) -> tuple[int, ...]:
"""
@@ -707,11 +641,11 @@ def ndim(self) -> int:
Examples
--------
- >>> s = pd.Series({'a': 1, 'b': 2, 'c': 3})
+ >>> s = pd.Series({"a": 1, "b": 2, "c": 3})
>>> s.ndim
1
- >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
+ >>> df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4]})
>>> df.ndim
2
"""
@@ -732,11 +666,11 @@ def size(self) -> int:
Examples
--------
- >>> s = pd.Series({'a': 1, 'b': 2, 'c': 3})
+ >>> s = pd.Series({"a": 1, "b": 2, "c": 3})
>>> s.size
3
- >>> df = pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
+ >>> df = pd.DataFrame({"col1": [1, 2], "col2": [3, 4]})
>>> df.size
4
"""
@@ -748,7 +682,7 @@ def set_axis(
labels,
*,
axis: Axis = 0,
- copy: bool_t | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
) -> Self:
"""
Assign desired index to given axis.
@@ -765,7 +699,7 @@ def set_axis(
The axis to update. The value 0 identifies the rows. For `Series`
this parameter is unused and defaults to 0.
- copy : bool, default True
+ copy : bool, default False
Whether to make a copy of the underlying data.
.. note::
@@ -780,6 +714,8 @@ def set_axis(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+ .. deprecated:: 3.0.0
+
Returns
-------
%(klass)s
@@ -789,20 +725,28 @@ def set_axis(
--------
%(klass)s.rename_axis : Alter the name of the index%(see_also_sub)s.
"""
- return self._set_axis_nocheck(labels, axis, inplace=False, copy=copy)
+ self._check_copy_deprecation(copy)
+ return self._set_axis_nocheck(labels, axis, inplace=False)
- @final
+ @overload
def _set_axis_nocheck(
- self, labels, axis: Axis, inplace: bool_t, copy: bool_t | None
- ):
+ self, labels, axis: Axis, inplace: Literal[False]
+ ) -> Self: ...
+
+ @overload
+ def _set_axis_nocheck(self, labels, axis: Axis, inplace: Literal[True]) -> None: ...
+
+ @overload
+ def _set_axis_nocheck(self, labels, axis: Axis, inplace: bool) -> Self | None: ...
+
+ @final
+ def _set_axis_nocheck(self, labels, axis: Axis, inplace: bool) -> Self | None:
if inplace:
setattr(self, self._get_axis_name(axis), labels)
- else:
- # With copy=False, we create a new object but don't copy the
- # underlying data.
- obj = self.copy(deep=copy and not using_copy_on_write())
- setattr(obj, obj._get_axis_name(axis), labels)
- return obj
+ return None
+ obj = self.copy(deep=False)
+ setattr(obj, obj._get_axis_name(axis), labels)
+ return obj
@final
def _set_axis(self, axis: AxisInt, labels: AnyArrayLike | list) -> None:
@@ -812,71 +756,6 @@ def _set_axis(self, axis: AxisInt, labels: AnyArrayLike | list) -> None:
"""
labels = ensure_index(labels)
self._mgr.set_axis(axis, labels)
- self._clear_item_cache()
-
- @final
- def swapaxes(self, axis1: Axis, axis2: Axis, copy: bool_t | None = None) -> Self:
- """
- Interchange axes and swap values axes appropriately.
-
- .. deprecated:: 2.1.0
- ``swapaxes`` is deprecated and will be removed.
- Please use ``transpose`` instead.
-
- Returns
- -------
- same as input
-
- Examples
- --------
- Please see examples for :meth:`DataFrame.transpose`.
- """
- warnings.warn(
- # GH#51946
- f"'{type(self).__name__}.swapaxes' is deprecated and "
- "will be removed in a future version. "
- f"Please use '{type(self).__name__}.transpose' instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- i = self._get_axis_number(axis1)
- j = self._get_axis_number(axis2)
-
- if i == j:
- return self.copy(deep=copy and not using_copy_on_write())
-
- mapping = {i: j, j: i}
-
- new_axes = [self._get_axis(mapping.get(k, k)) for k in range(self._AXIS_LEN)]
- new_values = self._values.swapaxes(i, j) # type: ignore[union-attr]
- if self._mgr.is_single_block and isinstance(self._mgr, BlockManager):
- # This should only get hit in case of having a single block, otherwise a
- # copy is made, we don't have to set up references.
- new_mgr = ndarray_to_mgr(
- new_values,
- new_axes[0],
- new_axes[1],
- dtype=None,
- copy=False,
- typ="block",
- )
- assert isinstance(new_mgr, BlockManager)
- assert isinstance(self._mgr, BlockManager)
- new_mgr.blocks[0].refs = self._mgr.blocks[0].refs
- new_mgr.blocks[0].refs.add_reference(new_mgr.blocks[0])
- if not using_copy_on_write() and copy is not False:
- new_mgr = new_mgr.copy(deep=True)
-
- out = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
- return out.__finalize__(self, method="swapaxes")
-
- return self._constructor(
- new_values,
- *new_axes,
- # The no-copy case for CoW is handled above
- copy=False,
- ).__finalize__(self, method="swapaxes")
@final
@doc(klass=_shared_doc_kwargs["klass"])
@@ -906,15 +785,15 @@ def droplevel(self, level: IndexLabel, axis: Axis = 0) -> Self:
Examples
--------
- >>> df = pd.DataFrame([
- ... [1, 2, 3, 4],
- ... [5, 6, 7, 8],
- ... [9, 10, 11, 12]
- ... ]).set_index([0, 1]).rename_axis(['a', 'b'])
+ >>> df = (
+ ... pd.DataFrame([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])
+ ... .set_index([0, 1])
+ ... .rename_axis(["a", "b"])
+ ... )
- >>> df.columns = pd.MultiIndex.from_tuples([
- ... ('c', 'e'), ('d', 'f')
- ... ], names=['level_1', 'level_2'])
+ >>> df.columns = pd.MultiIndex.from_tuples(
+ ... [("c", "e"), ("d", "f")], names=["level_1", "level_2"]
+ ... )
>>> df
level_1 c d
@@ -924,7 +803,7 @@ def droplevel(self, level: IndexLabel, axis: Axis = 0) -> Self:
5 6 7 8
9 10 11 12
- >>> df.droplevel('a')
+ >>> df.droplevel("a")
level_1 c d
level_2 e f
b
@@ -932,7 +811,7 @@ def droplevel(self, level: IndexLabel, axis: Axis = 0) -> Self:
6 7 8
10 11 12
- >>> df.droplevel('level_2', axis=1)
+ >>> df.droplevel("level_2", axis=1)
level_1 c d
a b
1 2 3 4
@@ -941,7 +820,7 @@ def droplevel(self, level: IndexLabel, axis: Axis = 0) -> Self:
"""
labels = self._get_axis(axis)
new_labels = labels.droplevel(level)
- return self.set_axis(new_labels, axis=axis, copy=None)
+ return self.set_axis(new_labels, axis=axis)
def pop(self, item: Hashable) -> Series | Any:
result = self[item]
@@ -1012,7 +891,7 @@ def squeeze(self, axis: Axis | None = None):
Squeezing is even more effective when used with DataFrames.
- >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=['a', 'b'])
+ >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["a", "b"])
>>> df
a b
0 1 2
@@ -1021,7 +900,7 @@ def squeeze(self, axis: Axis | None = None):
Slicing a single column will produce a DataFrame with the columns
having only one value:
- >>> df_a = df[['a']]
+ >>> df_a = df[["a"]]
>>> df_a
a
0 1
@@ -1029,7 +908,7 @@ def squeeze(self, axis: Axis | None = None):
So the columns can be squeezed down, resulting in a Series:
- >>> df_a.squeeze('columns')
+ >>> df_a.squeeze("columns")
0 1
1 3
Name: a, dtype: int64
@@ -1037,14 +916,14 @@ def squeeze(self, axis: Axis | None = None):
Slicing a single row from a single column will produce a single
scalar DataFrame:
- >>> df_0a = df.loc[df.index < 1, ['a']]
+ >>> df_0a = df.loc[df.index < 1, ["a"]]
>>> df_0a
a
0 1
Squeezing the rows produces a single scalar Series:
- >>> df_0a.squeeze('rows')
+ >>> df_0a.squeeze("rows")
a 1
Name: 0, dtype: int64
@@ -1067,6 +946,45 @@ def squeeze(self, axis: Axis | None = None):
# ----------------------------------------------------------------------
# Rename
+ @overload
+ def _rename(
+ self,
+ mapper: Renamer | None = ...,
+ *,
+ index: Renamer | None = ...,
+ columns: Renamer | None = ...,
+ axis: Axis | None = ...,
+ inplace: Literal[False] = ...,
+ level: Level | None = ...,
+ errors: str = ...,
+ ) -> Self: ...
+
+ @overload
+ def _rename(
+ self,
+ mapper: Renamer | None = ...,
+ *,
+ index: Renamer | None = ...,
+ columns: Renamer | None = ...,
+ axis: Axis | None = ...,
+ inplace: Literal[True],
+ level: Level | None = ...,
+ errors: str = ...,
+ ) -> None: ...
+
+ @overload
+ def _rename(
+ self,
+ mapper: Renamer | None = ...,
+ *,
+ index: Renamer | None = ...,
+ columns: Renamer | None = ...,
+ axis: Axis | None = ...,
+ inplace: bool,
+ level: Level | None = ...,
+ errors: str = ...,
+ ) -> Self | None: ...
+
@final
def _rename(
self,
@@ -1075,8 +993,7 @@ def _rename(
index: Renamer | None = None,
columns: Renamer | None = None,
axis: Axis | None = None,
- copy: bool_t | None = None,
- inplace: bool_t = False,
+ inplace: bool = False,
level: Level | None = None,
errors: str = "ignore",
) -> Self | None:
@@ -1102,7 +1019,7 @@ def _rename(
index = mapper
self._check_inplace_and_allows_duplicate_labels(inplace)
- result = self if inplace else self.copy(deep=copy and not using_copy_on_write())
+ result = self if inplace else self.copy(deep=False)
for axis_no, replacements in enumerate((index, columns)):
if replacements is None:
@@ -1130,8 +1047,7 @@ def _rename(
raise KeyError(f"{missing_labels} not found in axis")
new_index = ax._transform_index(f, level=level)
- result._set_axis_nocheck(new_index, axis=axis_no, inplace=True, copy=False)
- result._clear_item_cache()
+ result._set_axis_nocheck(new_index, axis=axis_no, inplace=True)
if inplace:
self._update_inplace(result)
@@ -1147,10 +1063,9 @@ def rename_axis(
index=...,
columns=...,
axis: Axis = ...,
- copy: bool_t | None = ...,
+ copy: bool | lib.NoDefault = lib.no_default,
inplace: Literal[False] = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def rename_axis(
@@ -1160,10 +1075,9 @@ def rename_axis(
index=...,
columns=...,
axis: Axis = ...,
- copy: bool_t | None = ...,
+ copy: bool | lib.NoDefault = lib.no_default,
inplace: Literal[True],
- ) -> None:
- ...
+ ) -> None: ...
@overload
def rename_axis(
@@ -1173,10 +1087,9 @@ def rename_axis(
index=...,
columns=...,
axis: Axis = ...,
- copy: bool_t | None = ...,
- inplace: bool_t = ...,
- ) -> Self | None:
- ...
+ copy: bool | lib.NoDefault = lib.no_default,
+ inplace: bool = ...,
+ ) -> Self | None: ...
def rename_axis(
self,
@@ -1185,8 +1098,8 @@ def rename_axis(
index=lib.no_default,
columns=lib.no_default,
axis: Axis = 0,
- copy: bool_t | None = None,
- inplace: bool_t = False,
+ copy: bool | lib.NoDefault = lib.no_default,
+ inplace: bool = False,
) -> Self | None:
"""
Set the name of the axis for the index or columns.
@@ -1195,19 +1108,19 @@ def rename_axis(
----------
mapper : scalar, list-like, optional
Value to set the axis name attribute.
- index, columns : scalar, list-like, dict-like or function, optional
- A scalar, list-like, dict-like or functions transformations to
- apply to that axis' values.
- Note that the ``columns`` parameter is not allowed if the
- object is a Series. This parameter only apply for DataFrame
- type objects.
Use either ``mapper`` and ``axis`` to
specify the axis to target with ``mapper``, or ``index``
and/or ``columns``.
+ index : scalar, list-like, dict-like or function, optional
+ A scalar, list-like, dict-like or functions transformations to
+ apply to that axis' values.
+ columns : scalar, list-like, dict-like or function, optional
+ A scalar, list-like, dict-like or functions transformations to
+ apply to that axis' values.
axis : {0 or 'index', 1 or 'columns'}, default 0
- The axis to rename. For `Series` this parameter is unused and defaults to 0.
- copy : bool, default None
+ The axis to rename.
+ copy : bool, default False
Also copy underlying data.
.. note::
@@ -1221,13 +1134,15 @@ def rename_axis(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+
+ .. deprecated:: 3.0.0
inplace : bool, default False
Modifies the object directly, instead of creating a new Series
or DataFrame.
Returns
-------
- Series, DataFrame, or None
+ DataFrame, or None
The same type as the caller or None if ``inplace=True``.
See Also
@@ -1257,26 +1172,11 @@ def rename_axis(
Examples
--------
- **Series**
-
- >>> s = pd.Series(["dog", "cat", "monkey"])
- >>> s
- 0 dog
- 1 cat
- 2 monkey
- dtype: object
- >>> s.rename_axis("animal")
- animal
- 0 dog
- 1 cat
- 2 monkey
- dtype: object
-
**DataFrame**
- >>> df = pd.DataFrame({"num_legs": [4, 4, 2],
- ... "num_arms": [0, 0, 2]},
- ... ["dog", "cat", "monkey"])
+ >>> df = pd.DataFrame(
+ ... {"num_legs": [4, 4, 2], "num_arms": [0, 0, 2]}, ["dog", "cat", "monkey"]
+ ... )
>>> df
num_legs num_arms
dog 4 0
@@ -1299,9 +1199,9 @@ def rename_axis(
**MultiIndex**
- >>> df.index = pd.MultiIndex.from_product([['mammal'],
- ... ['dog', 'cat', 'monkey']],
- ... names=['type', 'name'])
+ >>> df.index = pd.MultiIndex.from_product(
+ ... [["mammal"], ["dog", "cat", "monkey"]], names=["type", "name"]
+ ... )
>>> df
limbs num_legs num_arms
type name
@@ -1309,7 +1209,7 @@ def rename_axis(
cat 4 0
monkey 2 2
- >>> df.rename_axis(index={'type': 'class'})
+ >>> df.rename_axis(index={"type": "class"})
limbs num_legs num_arms
class name
mammal dog 4 0
@@ -1323,6 +1223,7 @@ class name
cat 4 0
monkey 2 2
"""
+ self._check_copy_deprecation(copy)
axes = {"index": index, "columns": columns}
if axis is not None:
@@ -1330,24 +1231,19 @@ class name
inplace = validate_bool_kwarg(inplace, "inplace")
- if copy and using_copy_on_write():
- copy = False
-
if mapper is not lib.no_default:
# Use v0.23 behavior if a scalar or list
non_mapper = is_scalar(mapper) or (
is_list_like(mapper) and not is_dict_like(mapper)
)
if non_mapper:
- return self._set_axis_name(
- mapper, axis=axis, inplace=inplace, copy=copy
- )
+ return self._set_axis_name(mapper, axis=axis, inplace=inplace)
else:
raise ValueError("Use `.rename` to alter labels with a mapper.")
else:
# Use new behavior. Means that index and/or columns
# is specified
- result = self if inplace else self.copy(deep=copy)
+ result = self if inplace else self.copy(deep=False)
for axis in range(self._AXIS_LEN):
v = axes.get(self._get_axis_name(axis))
@@ -1360,15 +1256,30 @@ class name
f = common.get_rename_function(v)
curnames = self._get_axis(axis).names
newnames = [f(name) for name in curnames]
- result._set_axis_name(newnames, axis=axis, inplace=True, copy=copy)
+ result._set_axis_name(newnames, axis=axis, inplace=True)
if not inplace:
return result
return None
+ @overload
+ def _set_axis_name(
+ self, name, axis: Axis = ..., *, inplace: Literal[False] = ...
+ ) -> Self: ...
+
+ @overload
+ def _set_axis_name(
+ self, name, axis: Axis = ..., *, inplace: Literal[True]
+ ) -> None: ...
+
+ @overload
+ def _set_axis_name(
+ self, name, axis: Axis = ..., *, inplace: bool
+ ) -> Self | None: ...
+
@final
def _set_axis_name(
- self, name, axis: Axis = 0, inplace: bool_t = False, copy: bool_t | None = True
- ):
+ self, name, axis: Axis = 0, *, inplace: bool = False
+ ) -> Self | None:
"""
Set the name(s) of the axis.
@@ -1381,8 +1292,6 @@ def _set_axis_name(
and the value 1 or 'columns' specifies columns.
inplace : bool, default False
If `True`, do operation inplace and return None.
- copy:
- Whether to make a copy of the result.
Returns
-------
@@ -1398,8 +1307,7 @@ def _set_axis_name(
Examples
--------
- >>> df = pd.DataFrame({"num_legs": [4, 4, 2]},
- ... ["dog", "cat", "monkey"])
+ >>> df = pd.DataFrame({"num_legs": [4, 4, 2]}, ["dog", "cat", "monkey"])
>>> df
num_legs
dog 4
@@ -1412,7 +1320,8 @@ def _set_axis_name(
cat 4
monkey 2
>>> df.index = pd.MultiIndex.from_product(
- ... [["mammal"], ['dog', 'cat', 'monkey']])
+ ... [["mammal"], ["dog", "cat", "monkey"]]
+ ... )
>>> df._set_axis_name(["type", "name"])
num_legs
type name
@@ -1424,7 +1333,7 @@ def _set_axis_name(
idx = self._get_axis(axis).set_names(name)
inplace = validate_bool_kwarg(inplace, "inplace")
- renamed = self if inplace else self.copy(deep=copy)
+ renamed = self if inplace else self.copy(deep=False)
if axis == 0:
renamed.index = idx
else:
@@ -1432,18 +1341,19 @@ def _set_axis_name(
if not inplace:
return renamed
+ return None
# ----------------------------------------------------------------------
# Comparison Methods
@final
- def _indexed_same(self, other) -> bool_t:
+ def _indexed_same(self, other) -> bool:
return all(
self._get_axis(a).equals(other._get_axis(a)) for a in self._AXIS_ORDERS
)
@final
- def equals(self, other: object) -> bool_t:
+ def equals(self, other: object) -> bool:
"""
Test whether two objects contain the same elements.
@@ -1581,73 +1491,6 @@ def __nonzero__(self) -> NoReturn:
__bool__ = __nonzero__
- @final
- def bool(self) -> bool_t:
- """
- Return the bool of a single element Series or DataFrame.
-
- .. deprecated:: 2.1.0
-
- bool is deprecated and will be removed in future version of pandas.
- For ``Series`` use ``pandas.Series.item``.
-
- This must be a boolean scalar value, either True or False. It will raise a
- ValueError if the Series or DataFrame does not have exactly 1 element, or that
- element is not boolean (integer values 0 and 1 will also raise an exception).
-
- Returns
- -------
- bool
- The value in the Series or DataFrame.
-
- See Also
- --------
- Series.astype : Change the data type of a Series, including to boolean.
- DataFrame.astype : Change the data type of a DataFrame, including to boolean.
- numpy.bool_ : NumPy boolean data type, used by pandas for boolean values.
-
- Examples
- --------
- The method will only work for single element objects with a boolean value:
-
- >>> pd.Series([True]).bool() # doctest: +SKIP
- True
- >>> pd.Series([False]).bool() # doctest: +SKIP
- False
-
- >>> pd.DataFrame({'col': [True]}).bool() # doctest: +SKIP
- True
- >>> pd.DataFrame({'col': [False]}).bool() # doctest: +SKIP
- False
-
- This is an alternative method and will only work
- for single element objects with a boolean value:
-
- >>> pd.Series([True]).item() # doctest: +SKIP
- True
- >>> pd.Series([False]).item() # doctest: +SKIP
- False
- """
-
- warnings.warn(
- f"{type(self).__name__}.bool is now deprecated and will be removed "
- "in future version of pandas",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- v = self.squeeze()
- if isinstance(v, (bool, np.bool_)):
- return bool(v)
- elif is_scalar(v):
- raise ValueError(
- "bool cannot act on a non-boolean single element "
- f"{type(self).__name__}"
- )
-
- self.__nonzero__()
- # for mypy (__nonzero__ raises)
- return True
-
@final
def abs(self) -> Self:
"""
@@ -1690,7 +1533,7 @@ def abs(self) -> Self:
Absolute numeric values in a Series with a Timedelta element.
- >>> s = pd.Series([pd.Timedelta('1 days')])
+ >>> s = pd.Series([pd.Timedelta("1 days")])
>>> s.abs()
0 1 days
dtype: timedelta64[ns]
@@ -1698,11 +1541,9 @@ def abs(self) -> Self:
Select rows with data closest to certain value using argsort (from
`StackOverflow `__).
- >>> df = pd.DataFrame({
- ... 'a': [4, 5, 6, 7],
- ... 'b': [10, 20, 30, 40],
- ... 'c': [100, 50, -30, -50]
- ... })
+ >>> df = pd.DataFrame(
+ ... {"a": [4, 5, 6, 7], "b": [10, 20, 30, 40], "c": [100, 50, -30, -50]}
+ ... )
>>> df
a b c
0 4 10 100
@@ -1738,7 +1579,7 @@ def __round__(self, decimals: int = 0) -> Self:
# have consistent precedence and validation logic throughout the library.
@final
- def _is_level_reference(self, key: Level, axis: Axis = 0) -> bool_t:
+ def _is_level_reference(self, key: Level, axis: Axis = 0) -> bool:
"""
Test whether a key is a level reference for a given axis.
@@ -1769,7 +1610,7 @@ def _is_level_reference(self, key: Level, axis: Axis = 0) -> bool_t:
)
@final
- def _is_label_reference(self, key: Level, axis: Axis = 0) -> bool_t:
+ def _is_label_reference(self, key: Level, axis: Axis = 0) -> bool:
"""
Test whether a key is a label reference for a given axis.
@@ -1799,7 +1640,7 @@ def _is_label_reference(self, key: Level, axis: Axis = 0) -> bool_t:
)
@final
- def _is_label_or_level_reference(self, key: Level, axis: AxisInt = 0) -> bool_t:
+ def _is_label_or_level_reference(self, key: Level, axis: AxisInt = 0) -> bool:
"""
Test whether a key is a label or level reference for a given axis.
@@ -2023,7 +1864,7 @@ def __iter__(self) -> Iterator:
Examples
--------
- >>> df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
+ >>> df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
>>> for x in df:
... print(x)
A
@@ -2045,8 +1886,9 @@ def keys(self) -> Index:
Examples
--------
- >>> d = pd.DataFrame(data={'A': [1, 2, 3], 'B': [0, 4, 8]},
- ... index=['a', 'b', 'c'])
+ >>> d = pd.DataFrame(
+ ... data={"A": [1, 2, 3], "B": [0, 4, 8]}, index=["a", "b", "c"]
+ ... )
>>> d
A B
a 1 0
@@ -2075,12 +1917,12 @@ def __len__(self) -> int:
return len(self._info_axis)
@final
- def __contains__(self, key) -> bool_t:
+ def __contains__(self, key) -> bool:
"""True if the key is in the info axis"""
return key in self._info_axis
@property
- def empty(self) -> bool_t:
+ def empty(self) -> bool:
"""
Indicator whether Series/DataFrame is empty.
@@ -2107,7 +1949,7 @@ def empty(self) -> bool_t:
--------
An example of an actual empty DataFrame. Notice the index is empty:
- >>> df_empty = pd.DataFrame({'A' : []})
+ >>> df_empty = pd.DataFrame({"A": []})
>>> df_empty
Empty DataFrame
Columns: [A]
@@ -2118,7 +1960,7 @@ def empty(self) -> bool_t:
If we only have NaNs in our DataFrame, it is not considered empty! We
will need to drop the NaNs to make the DataFrame empty:
- >>> df = pd.DataFrame({'A' : [np.nan]})
+ >>> df = pd.DataFrame({"A": [np.nan]})
>>> df
A
0 NaN
@@ -2127,7 +1969,7 @@ def empty(self) -> bool_t:
>>> df.dropna().empty
True
- >>> ser_empty = pd.Series({'A' : []})
+ >>> ser_empty = pd.Series({"A": []})
>>> ser_empty
A []
dtype: object
@@ -2146,14 +1988,12 @@ def empty(self) -> bool_t:
# GH#23114 Ensure ndarray.__op__(DataFrame) returns NotImplemented
__array_priority__: int = 1000
- def __array__(self, dtype: npt.DTypeLike | None = None) -> np.ndarray:
+ def __array__(
+ self, dtype: npt.DTypeLike | None = None, copy: bool | None = None
+ ) -> np.ndarray:
values = self._values
arr = np.asarray(values, dtype=dtype)
- if (
- astype_is_view(values.dtype, arr.dtype)
- and using_copy_on_write()
- and self._mgr.is_single_block
- ):
+ if astype_is_view(values.dtype, arr.dtype) and self._mgr.is_single_block:
# Check if both conversions can be done without a copy
if astype_is_view(self.dtypes.iloc[0], values.dtype) and astype_is_view(
values.dtype, arr.dtype
@@ -2219,8 +2059,6 @@ def __setstate__(self, state) -> None:
elif len(state) == 2:
raise NotImplementedError("Pre-0.12 pickles are no longer supported")
- self._item_cache: dict[Hashable, Series] = {}
-
# ----------------------------------------------------------------------
# Rendering Methods
@@ -2258,28 +2096,32 @@ def _repr_data_resource_(self):
# I/O Methods
@final
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "excel_writer"], name="to_excel"
- )
@doc(
klass="object",
storage_options=_shared_docs["storage_options"],
storage_options_versionadded="1.2.0",
+ extra_parameters=textwrap.dedent(
+ """\
+ engine_kwargs : dict, optional
+ Arbitrary keyword arguments passed to excel engine.
+ """
+ ),
)
def to_excel(
self,
excel_writer: FilePath | WriteExcelBuffer | ExcelWriter,
+ *,
sheet_name: str = "Sheet1",
na_rep: str = "",
float_format: str | None = None,
columns: Sequence[Hashable] | None = None,
- header: Sequence[Hashable] | bool_t = True,
- index: bool_t = True,
+ header: Sequence[Hashable] | bool = True,
+ index: bool = True,
index_label: IndexLabel | None = None,
startrow: int = 0,
startcol: int = 0,
engine: Literal["openpyxl", "xlsxwriter"] | None = None,
- merge_cells: bool_t = True,
+ merge_cells: bool = True,
inf_rep: str = "inf",
freeze_panes: tuple[int, int] | None = None,
storage_options: StorageOptions | None = None,
@@ -2340,9 +2182,7 @@ def to_excel(
{storage_options}
.. versionadded:: {storage_options_versionadded}
- engine_kwargs : dict, optional
- Arbitrary keyword arguments passed to excel engine.
-
+ {extra_parameters}
See Also
--------
to_csv : Write DataFrame to a comma-separated values (csv) file.
@@ -2359,40 +2199,44 @@ def to_excel(
Once a workbook has been saved it is not possible to write further
data without rewriting the whole workbook.
+ pandas will check the number of rows, columns,
+ and cell character count does not exceed Excel's limitations.
+ All other limitations must be checked by the user.
+
Examples
--------
Create, write to and save a workbook:
- >>> df1 = pd.DataFrame([['a', 'b'], ['c', 'd']],
- ... index=['row 1', 'row 2'],
- ... columns=['col 1', 'col 2'])
+ >>> df1 = pd.DataFrame(
+ ... [["a", "b"], ["c", "d"]],
+ ... index=["row 1", "row 2"],
+ ... columns=["col 1", "col 2"],
+ ... )
>>> df1.to_excel("output.xlsx") # doctest: +SKIP
To specify the sheet name:
- >>> df1.to_excel("output.xlsx",
- ... sheet_name='Sheet_name_1') # doctest: +SKIP
+ >>> df1.to_excel("output.xlsx", sheet_name="Sheet_name_1") # doctest: +SKIP
If you wish to write to more than one sheet in the workbook, it is
necessary to specify an ExcelWriter object:
>>> df2 = df1.copy()
- >>> with pd.ExcelWriter('output.xlsx') as writer: # doctest: +SKIP
- ... df1.to_excel(writer, sheet_name='Sheet_name_1')
- ... df2.to_excel(writer, sheet_name='Sheet_name_2')
+ >>> with pd.ExcelWriter("output.xlsx") as writer: # doctest: +SKIP
+ ... df1.to_excel(writer, sheet_name="Sheet_name_1")
+ ... df2.to_excel(writer, sheet_name="Sheet_name_2")
ExcelWriter can also be used to append to an existing Excel file:
- >>> with pd.ExcelWriter('output.xlsx',
- ... mode='a') as writer: # doctest: +SKIP
- ... df1.to_excel(writer, sheet_name='Sheet_name_3')
+ >>> with pd.ExcelWriter("output.xlsx", mode="a") as writer: # doctest: +SKIP
+ ... df1.to_excel(writer, sheet_name="Sheet_name_3")
To set the library that is used to write the Excel file,
you can pass the `engine` keyword (the default engine is
automatically chosen depending on the file extension):
- >>> df1.to_excel('output1.xlsx', engine='xlsxwriter') # doctest: +SKIP
+ >>> df1.to_excel("output1.xlsx", engine="xlsxwriter") # doctest: +SKIP
"""
if engine_kwargs is None:
engine_kwargs = {}
@@ -2424,9 +2268,6 @@ def to_excel(
)
@final
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "path_or_buf"], name="to_json"
- )
@doc(
storage_options=_shared_docs["storage_options"],
compression_options=_shared_docs["compression_options"] % "path_or_buf",
@@ -2434,16 +2275,17 @@ def to_excel(
def to_json(
self,
path_or_buf: FilePath | WriteBuffer[bytes] | WriteBuffer[str] | None = None,
+ *,
orient: Literal["split", "records", "index", "table", "columns", "values"]
| None = None,
date_format: str | None = None,
double_precision: int = 10,
- force_ascii: bool_t = True,
+ force_ascii: bool = True,
date_unit: TimeUnit = "ms",
default_handler: Callable[[Any], JSONSerializable] | None = None,
- lines: bool_t = False,
+ lines: bool = False,
compression: CompressionOptions = "infer",
- index: bool_t | None = None,
+ index: bool | None = None,
indent: int | None = None,
storage_options: StorageOptions | None = None,
mode: Literal["a", "w"] = "w",
@@ -2715,22 +2557,20 @@ def to_json(
)
@final
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "path_or_buf"], name="to_hdf"
- )
def to_hdf(
self,
path_or_buf: FilePath | HDFStore,
+ *,
key: str,
mode: Literal["a", "w", "r+"] = "a",
complevel: int | None = None,
complib: Literal["zlib", "lzo", "bzip2", "blosc"] | None = None,
- append: bool_t = False,
+ append: bool = False,
format: Literal["fixed", "table"] | None = None,
- index: bool_t = True,
+ index: bool = True,
min_itemsize: int | dict[str, int] | None = None,
nan_rep=None,
- dropna: bool_t | None = None,
+ dropna: bool | None = None,
data_columns: Literal[True] | list[str] | None = None,
errors: OpenFileErrors = "strict",
encoding: str = "UTF-8",
@@ -2811,6 +2651,7 @@ def to_hdf(
See the errors argument for :func:`open` for a full list
of options.
encoding : str, default "UTF-8"
+ Set character encoding.
See Also
--------
@@ -2823,23 +2664,24 @@ def to_hdf(
Examples
--------
- >>> df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]},
- ... index=['a', 'b', 'c']) # doctest: +SKIP
- >>> df.to_hdf('data.h5', key='df', mode='w') # doctest: +SKIP
+ >>> df = pd.DataFrame(
+ ... {"A": [1, 2, 3], "B": [4, 5, 6]}, index=["a", "b", "c"]
+ ... ) # doctest: +SKIP
+ >>> df.to_hdf("data.h5", key="df", mode="w") # doctest: +SKIP
We can add another object to the same file:
>>> s = pd.Series([1, 2, 3, 4]) # doctest: +SKIP
- >>> s.to_hdf('data.h5', key='s') # doctest: +SKIP
+ >>> s.to_hdf("data.h5", key="s") # doctest: +SKIP
Reading from HDF file:
- >>> pd.read_hdf('data.h5', 'df') # doctest: +SKIP
+ >>> pd.read_hdf("data.h5", "df") # doctest: +SKIP
A B
a 1 4
b 2 5
c 3 6
- >>> pd.read_hdf('data.h5', 's') # doctest: +SKIP
+ >>> pd.read_hdf("data.h5", "s") # doctest: +SKIP
0 1
1 2
2 3
@@ -2869,16 +2711,14 @@ def to_hdf(
)
@final
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "name", "con"], name="to_sql"
- )
def to_sql(
self,
name: str,
con,
+ *,
schema: str | None = None,
if_exists: Literal["fail", "replace", "append"] = "fail",
- index: bool_t = True,
+ index: bool = True,
index_label: IndexLabel | None = None,
chunksize: int | None = None,
dtype: DtypeArg | None = None,
@@ -2970,6 +2810,9 @@ def to_sql(
database. Otherwise, the datetimes will be stored as timezone unaware
timestamps local to the original timezone.
+ Not all datastores support ``method="multi"``. Oracle, for example,
+ does not support multi-value insert.
+
References
----------
.. [1] https://docs.sqlalchemy.org
@@ -2995,7 +2838,7 @@ def to_sql(
3
>>> from sqlalchemy import text
>>> with engine.connect() as conn:
- ... conn.execute(text("SELECT * FROM users")).fetchall()
+ ... conn.execute(text("SELECT * FROM users")).fetchall()
[(0, 'User 1'), (1, 'User 2'), (2, 'User 3')]
An `sqlalchemy.engine.Connection` can also be passed to `con`:
@@ -3012,7 +2855,7 @@ def to_sql(
>>> df2.to_sql(name='users', con=engine, if_exists='append')
2
>>> with engine.connect() as conn:
- ... conn.execute(text("SELECT * FROM users")).fetchall()
+ ... conn.execute(text("SELECT * FROM users")).fetchall()
[(0, 'User 1'), (1, 'User 2'), (2, 'User 3'),
(0, 'User 4'), (1, 'User 5'), (0, 'User 6'),
(1, 'User 7')]
@@ -3023,7 +2866,7 @@ def to_sql(
... index_label='id')
2
>>> with engine.connect() as conn:
- ... conn.execute(text("SELECT * FROM users")).fetchall()
+ ... conn.execute(text("SELECT * FROM users")).fetchall()
[(0, 'User 6'), (1, 'User 7')]
Use ``method`` to define a callable insertion method to do nothing
@@ -3036,13 +2879,14 @@ def to_sql(
... stmt = insert(table.table).values(data).on_conflict_do_nothing(index_elements=["a"])
... result = conn.execute(stmt)
... return result.rowcount
- >>> df_conflict.to_sql(name="conflict_table", con=conn, if_exists="append", method=insert_on_conflict_nothing) # doctest: +SKIP
+ >>> df_conflict.to_sql(name="conflict_table", con=conn, if_exists="append", # noqa: F821
+ ... method=insert_on_conflict_nothing) # doctest: +SKIP
0
For MySQL, a callable to update columns ``b`` and ``c`` if there's a conflict
on a primary key.
- >>> from sqlalchemy.dialects.mysql import insert
+ >>> from sqlalchemy.dialects.mysql import insert # noqa: F811
>>> def insert_on_conflict_update(table, conn, keys, data_iter):
... # update columns "b" and "c" on primary key conflict
... data = [dict(zip(keys, row)) for row in data_iter]
@@ -3053,7 +2897,8 @@ def to_sql(
... stmt = stmt.on_duplicate_key_update(b=stmt.inserted.b, c=stmt.inserted.c)
... result = conn.execute(stmt)
... return result.rowcount
- >>> df_conflict.to_sql(name="conflict_table", con=conn, if_exists="append", method=insert_on_conflict_update) # doctest: +SKIP
+ >>> df_conflict.to_sql(name="conflict_table", con=conn, if_exists="append", # noqa: F821
+ ... method=insert_on_conflict_update) # doctest: +SKIP
2
Specify the dtype (especially useful for integers with missing values).
@@ -3074,7 +2919,7 @@ def to_sql(
3
>>> with engine.connect() as conn:
- ... conn.execute(text("SELECT * FROM integers")).fetchall()
+ ... conn.execute(text("SELECT * FROM integers")).fetchall()
[(1,), (None,), (2,)]
""" # noqa: E501
from pandas.io import sql
@@ -3093,9 +2938,6 @@ def to_sql(
)
@final
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "path"], name="to_pickle"
- )
@doc(
storage_options=_shared_docs["storage_options"],
compression_options=_shared_docs["compression_options"] % "path",
@@ -3103,6 +2945,7 @@ def to_sql(
def to_pickle(
self,
path: FilePath | WriteBuffer[bytes],
+ *,
compression: CompressionOptions = "infer",
protocol: int = pickle.HIGHEST_PROTOCOL,
storage_options: StorageOptions | None = None,
@@ -3136,7 +2979,9 @@ def to_pickle(
Examples
--------
- >>> original_df = pd.DataFrame({{"foo": range(5), "bar": range(5, 10)}}) # doctest: +SKIP
+ >>> original_df = pd.DataFrame(
+ ... {{"foo": range(5), "bar": range(5, 10)}}
+ ... ) # doctest: +SKIP
>>> original_df # doctest: +SKIP
foo bar
0 0 5
@@ -3154,7 +2999,7 @@ def to_pickle(
2 2 7
3 3 8
4 4 9
- """ # noqa: E501
+ """
from pandas.io.pickle import to_pickle
to_pickle(
@@ -3166,11 +3011,8 @@ def to_pickle(
)
@final
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self"], name="to_clipboard"
- )
def to_clipboard(
- self, excel: bool_t = True, sep: str | None = None, **kwargs
+ self, *, excel: bool = True, sep: str | None = None, **kwargs
) -> None:
r"""
Copy object to the system clipboard.
@@ -3212,9 +3054,9 @@ def to_clipboard(
--------
Copy the contents of a DataFrame to the clipboard.
- >>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=['A', 'B', 'C'])
+ >>> df = pd.DataFrame([[1, 2, 3], [4, 5, 6]], columns=["A", "B", "C"])
- >>> df.to_clipboard(sep=',') # doctest: +SKIP
+ >>> df.to_clipboard(sep=",") # doctest: +SKIP
... # Wrote the following to the system clipboard:
... # ,A,B,C
... # 0,1,2,3
@@ -3223,7 +3065,7 @@ def to_clipboard(
We can omit the index by passing the keyword `index` and setting
it to false.
- >>> df.to_clipboard(sep=',', index=False) # doctest: +SKIP
+ >>> df.to_clipboard(sep=",", index=False) # doctest: +SKIP
... # Wrote the following to the system clipboard:
... # A,B,C
... # 1,2,3
@@ -3234,6 +3076,7 @@ def to_clipboard(
.. code-block:: python
import pyperclip
+
html = df.style.to_html()
pyperclip.copy(html)
"""
@@ -3263,12 +3106,15 @@ def to_xarray(self):
Examples
--------
- >>> df = pd.DataFrame([('falcon', 'bird', 389.0, 2),
- ... ('parrot', 'bird', 24.0, 2),
- ... ('lion', 'mammal', 80.5, 4),
- ... ('monkey', 'mammal', np.nan, 4)],
- ... columns=['name', 'class', 'max_speed',
- ... 'num_legs'])
+ >>> df = pd.DataFrame(
+ ... [
+ ... ("falcon", "bird", 389.0, 2),
+ ... ("parrot", "bird", 24.0, 2),
+ ... ("lion", "mammal", 80.5, 4),
+ ... ("monkey", "mammal", np.nan, 4),
+ ... ],
+ ... columns=["name", "class", "max_speed", "num_legs"],
+ ... )
>>> df
name class max_speed num_legs
0 falcon bird 389.0 2
@@ -3276,30 +3122,34 @@ def to_xarray(self):
2 lion mammal 80.5 4
3 monkey mammal NaN 4
- >>> df.to_xarray()
+ >>> df.to_xarray() # doctest: +SKIP
Dimensions: (index: 4)
Coordinates:
- * index (index) int64 0 1 2 3
+ * index (index) int64 32B 0 1 2 3
Data variables:
- name (index) object 'falcon' 'parrot' 'lion' 'monkey'
- class (index) object 'bird' 'bird' 'mammal' 'mammal'
- max_speed (index) float64 389.0 24.0 80.5 nan
- num_legs (index) int64 2 2 4 4
+ name (index) object 32B 'falcon' 'parrot' 'lion' 'monkey'
+ class (index) object 32B 'bird' 'bird' 'mammal' 'mammal'
+ max_speed (index) float64 32B 389.0 24.0 80.5 nan
+ num_legs (index) int64 32B 2 2 4 4
- >>> df['max_speed'].to_xarray()
+ >>> df["max_speed"].to_xarray() # doctest: +SKIP
array([389. , 24. , 80.5, nan])
Coordinates:
* index (index) int64 0 1 2 3
- >>> dates = pd.to_datetime(['2018-01-01', '2018-01-01',
- ... '2018-01-02', '2018-01-02'])
- >>> df_multiindex = pd.DataFrame({'date': dates,
- ... 'animal': ['falcon', 'parrot',
- ... 'falcon', 'parrot'],
- ... 'speed': [350, 18, 361, 15]})
- >>> df_multiindex = df_multiindex.set_index(['date', 'animal'])
+ >>> dates = pd.to_datetime(
+ ... ["2018-01-01", "2018-01-01", "2018-01-02", "2018-01-02"]
+ ... )
+ >>> df_multiindex = pd.DataFrame(
+ ... {
+ ... "date": dates,
+ ... "animal": ["falcon", "parrot", "falcon", "parrot"],
+ ... "speed": [350, 18, 361, 15],
+ ... }
+ ... )
+ >>> df_multiindex = df_multiindex.set_index(["date", "animal"])
>>> df_multiindex
speed
@@ -3309,7 +3159,7 @@ class (index) object 'bird' 'bird' 'mammal' 'mammal'
2018-01-02 falcon 361
parrot 15
- >>> df_multiindex.to_xarray()
+ >>> df_multiindex.to_xarray() # doctest: +SKIP
Dimensions: (date: 2, animal: 2)
Coordinates:
@@ -3329,80 +3179,78 @@ class (index) object 'bird' 'bird' 'mammal' 'mammal'
def to_latex(
self,
buf: None = ...,
+ *,
columns: Sequence[Hashable] | None = ...,
- header: bool_t | SequenceNotStr[str] = ...,
- index: bool_t = ...,
+ header: bool | SequenceNotStr[str] = ...,
+ index: bool = ...,
na_rep: str = ...,
formatters: FormattersType | None = ...,
float_format: FloatFormatType | None = ...,
- sparsify: bool_t | None = ...,
- index_names: bool_t = ...,
- bold_rows: bool_t = ...,
+ sparsify: bool | None = ...,
+ index_names: bool = ...,
+ bold_rows: bool = ...,
column_format: str | None = ...,
- longtable: bool_t | None = ...,
- escape: bool_t | None = ...,
+ longtable: bool | None = ...,
+ escape: bool | None = ...,
encoding: str | None = ...,
decimal: str = ...,
- multicolumn: bool_t | None = ...,
+ multicolumn: bool | None = ...,
multicolumn_format: str | None = ...,
- multirow: bool_t | None = ...,
+ multirow: bool | None = ...,
caption: str | tuple[str, str] | None = ...,
label: str | None = ...,
position: str | None = ...,
- ) -> str:
- ...
+ ) -> str: ...
@overload
def to_latex(
self,
buf: FilePath | WriteBuffer[str],
+ *,
columns: Sequence[Hashable] | None = ...,
- header: bool_t | SequenceNotStr[str] = ...,
- index: bool_t = ...,
+ header: bool | SequenceNotStr[str] = ...,
+ index: bool = ...,
na_rep: str = ...,
formatters: FormattersType | None = ...,
float_format: FloatFormatType | None = ...,
- sparsify: bool_t | None = ...,
- index_names: bool_t = ...,
- bold_rows: bool_t = ...,
+ sparsify: bool | None = ...,
+ index_names: bool = ...,
+ bold_rows: bool = ...,
column_format: str | None = ...,
- longtable: bool_t | None = ...,
- escape: bool_t | None = ...,
+ longtable: bool | None = ...,
+ escape: bool | None = ...,
encoding: str | None = ...,
decimal: str = ...,
- multicolumn: bool_t | None = ...,
+ multicolumn: bool | None = ...,
multicolumn_format: str | None = ...,
- multirow: bool_t | None = ...,
+ multirow: bool | None = ...,
caption: str | tuple[str, str] | None = ...,
label: str | None = ...,
position: str | None = ...,
- ) -> None:
- ...
+ ) -> None: ...
@final
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "buf"], name="to_latex"
- )
def to_latex(
self,
buf: FilePath | WriteBuffer[str] | None = None,
+ *,
columns: Sequence[Hashable] | None = None,
- header: bool_t | SequenceNotStr[str] = True,
- index: bool_t = True,
+ header: bool | SequenceNotStr[str] = True,
+ index: bool = True,
na_rep: str = "NaN",
formatters: FormattersType | None = None,
float_format: FloatFormatType | None = None,
- sparsify: bool_t | None = None,
- index_names: bool_t = True,
- bold_rows: bool_t = False,
+ sparsify: bool | None = None,
+ index_names: bool = True,
+ bold_rows: bool = False,
column_format: str | None = None,
- longtable: bool_t | None = None,
- escape: bool_t | None = None,
+ longtable: bool | None = None,
+ escape: bool | None = None,
encoding: str | None = None,
decimal: str = ".",
- multicolumn: bool_t | None = None,
+ multicolumn: bool | None = None,
multicolumn_format: str | None = None,
- multirow: bool_t | None = None,
+ multirow: bool | None = None,
caption: str | tuple[str, str] | None = None,
label: str | None = None,
position: str | None = None,
@@ -3544,7 +3392,7 @@ def to_latex(
>>> print(df.to_latex(index=False,
... formatters={"name": str.upper},
... float_format="{:.1f}".format,
- ... )) # doctest: +SKIP
+ ... )) # doctest: +SKIP
\begin{tabular}{lrr}
\toprule
name & age & height \\
@@ -3739,12 +3587,13 @@ def _to_latex_via_styler(
def to_csv(
self,
path_or_buf: None = ...,
+ *,
sep: str = ...,
na_rep: str = ...,
float_format: str | Callable | None = ...,
columns: Sequence[Hashable] | None = ...,
- header: bool_t | list[str] = ...,
- index: bool_t = ...,
+ header: bool | list[str] = ...,
+ index: bool = ...,
index_label: IndexLabel | None = ...,
mode: str = ...,
encoding: str | None = ...,
@@ -3754,24 +3603,24 @@ def to_csv(
lineterminator: str | None = ...,
chunksize: int | None = ...,
date_format: str | None = ...,
- doublequote: bool_t = ...,
+ doublequote: bool = ...,
escapechar: str | None = ...,
decimal: str = ...,
errors: OpenFileErrors = ...,
storage_options: StorageOptions = ...,
- ) -> str:
- ...
+ ) -> str: ...
@overload
def to_csv(
self,
path_or_buf: FilePath | WriteBuffer[bytes] | WriteBuffer[str],
+ *,
sep: str = ...,
na_rep: str = ...,
float_format: str | Callable | None = ...,
columns: Sequence[Hashable] | None = ...,
- header: bool_t | list[str] = ...,
- index: bool_t = ...,
+ header: bool | list[str] = ...,
+ index: bool = ...,
index_label: IndexLabel | None = ...,
mode: str = ...,
encoding: str | None = ...,
@@ -3781,18 +3630,14 @@ def to_csv(
lineterminator: str | None = ...,
chunksize: int | None = ...,
date_format: str | None = ...,
- doublequote: bool_t = ...,
+ doublequote: bool = ...,
escapechar: str | None = ...,
decimal: str = ...,
errors: OpenFileErrors = ...,
storage_options: StorageOptions = ...,
- ) -> None:
- ...
+ ) -> None: ...
@final
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "path_or_buf"], name="to_csv"
- )
@doc(
storage_options=_shared_docs["storage_options"],
compression_options=_shared_docs["compression_options"] % "path_or_buf",
@@ -3800,12 +3645,13 @@ def to_csv(
def to_csv(
self,
path_or_buf: FilePath | WriteBuffer[bytes] | WriteBuffer[str] | None = None,
+ *,
sep: str = ",",
na_rep: str = "",
float_format: str | Callable | None = None,
columns: Sequence[Hashable] | None = None,
- header: bool_t | list[str] = True,
- index: bool_t = True,
+ header: bool | list[str] = True,
+ index: bool = True,
index_label: IndexLabel | None = None,
mode: str = "w",
encoding: str | None = None,
@@ -3815,7 +3661,7 @@ def to_csv(
lineterminator: str | None = None,
chunksize: int | None = None,
date_format: str | None = None,
- doublequote: bool_t = True,
+ doublequote: bool = True,
escapechar: str | None = None,
decimal: str = ".",
errors: OpenFileErrors = "strict",
@@ -3922,31 +3768,34 @@ def to_csv(
--------
Create 'out.csv' containing 'df' without indices
- >>> df = pd.DataFrame({{'name': ['Raphael', 'Donatello'],
- ... 'mask': ['red', 'purple'],
- ... 'weapon': ['sai', 'bo staff']}})
- >>> df.to_csv('out.csv', index=False) # doctest: +SKIP
+ >>> df = pd.DataFrame(
+ ... [["Raphael", "red", "sai"], ["Donatello", "purple", "bo staff"]],
+ ... columns=["name", "mask", "weapon"],
+ ... )
+ >>> df.to_csv("out.csv", index=False) # doctest: +SKIP
Create 'out.zip' containing 'out.csv'
>>> df.to_csv(index=False)
'name,mask,weapon\nRaphael,red,sai\nDonatello,purple,bo staff\n'
- >>> compression_opts = dict(method='zip',
- ... archive_name='out.csv') # doctest: +SKIP
- >>> df.to_csv('out.zip', index=False,
- ... compression=compression_opts) # doctest: +SKIP
+ >>> compression_opts = dict(
+ ... method="zip", archive_name="out.csv"
+ ... ) # doctest: +SKIP
+ >>> df.to_csv(
+ ... "out.zip", index=False, compression=compression_opts
+ ... ) # doctest: +SKIP
To write a csv file to a new folder or nested folder you will first
need to create it using either Pathlib or os:
>>> from pathlib import Path # doctest: +SKIP
- >>> filepath = Path('folder/subfolder/out.csv') # doctest: +SKIP
+ >>> filepath = Path("folder/subfolder/out.csv") # doctest: +SKIP
>>> filepath.parent.mkdir(parents=True, exist_ok=True) # doctest: +SKIP
>>> df.to_csv(filepath) # doctest: +SKIP
>>> import os # doctest: +SKIP
- >>> os.makedirs('folder/subfolder', exist_ok=True) # doctest: +SKIP
- >>> df.to_csv('folder/subfolder/out.csv') # doctest: +SKIP
+ >>> os.makedirs("folder/subfolder", exist_ok=True) # doctest: +SKIP
+ >>> df.to_csv("folder/subfolder/out.csv") # doctest: +SKIP
"""
df = self if isinstance(self, ABCDataFrame) else self.to_frame()
@@ -3978,44 +3827,6 @@ def to_csv(
storage_options=storage_options,
)
- # ----------------------------------------------------------------------
- # Lookup Caching
-
- def _reset_cacher(self) -> None:
- """
- Reset the cacher.
- """
- raise AbstractMethodError(self)
-
- def _maybe_update_cacher(
- self,
- clear: bool_t = False,
- verify_is_copy: bool_t = True,
- inplace: bool_t = False,
- ) -> None:
- """
- See if we need to update our parent cacher if clear, then clear our
- cache.
-
- Parameters
- ----------
- clear : bool, default False
- Clear the item cache.
- verify_is_copy : bool, default True
- Provide is_copy checks.
- """
- if using_copy_on_write():
- return
-
- if verify_is_copy:
- self._check_setitem_copy(t="referent")
-
- if clear:
- self._clear_item_cache()
-
- def _clear_item_cache(self) -> None:
- raise AbstractMethodError(self)
-
# ----------------------------------------------------------------------
# Indexing Methods
@@ -4053,12 +3864,16 @@ def take(self, indices, axis: Axis = 0, **kwargs) -> Self:
Examples
--------
- >>> df = pd.DataFrame([('falcon', 'bird', 389.0),
- ... ('parrot', 'bird', 24.0),
- ... ('lion', 'mammal', 80.5),
- ... ('monkey', 'mammal', np.nan)],
- ... columns=['name', 'class', 'max_speed'],
- ... index=[0, 2, 3, 1])
+ >>> df = pd.DataFrame(
+ ... [
+ ... ("falcon", "bird", 389.0),
+ ... ("parrot", "bird", 24.0),
+ ... ("lion", "mammal", 80.5),
+ ... ("monkey", "mammal", np.nan),
+ ... ],
+ ... columns=["name", "class", "max_speed"],
+ ... index=[0, 2, 3, 1],
+ ... )
>>> df
name class max_speed
0 falcon bird 389.0
@@ -4097,33 +3912,14 @@ class max_speed
nv.validate_take((), kwargs)
- if not isinstance(indices, slice):
- indices = np.asarray(indices, dtype=np.intp)
- if (
- axis == 0
- and indices.ndim == 1
- and using_copy_on_write()
- and is_range_indexer(indices, len(self))
- ):
- return self.copy(deep=None)
- elif self.ndim == 1:
+ if isinstance(indices, slice):
raise TypeError(
f"{type(self).__name__}.take requires a sequence of integers, "
"not slice."
)
- else:
- warnings.warn(
- # GH#51539
- f"Passing a slice to {type(self).__name__}.take is deprecated "
- "and will raise in a future version. Use `obj[slicer]` or pass "
- "a sequence of integers instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- # We can get here with a slice via DataFrame.__getitem__
- indices = np.arange(
- indices.start, indices.stop, indices.step, dtype=np.intp
- )
+ indices = np.asarray(indices, dtype=np.intp)
+ if axis == 0 and indices.ndim == 1 and is_range_indexer(indices, len(self)):
+ return self.copy(deep=False)
new_data = self._mgr.take(
indices,
@@ -4134,30 +3930,13 @@ class max_speed
self, method="take"
)
- @final
- def _take_with_is_copy(self, indices, axis: Axis = 0) -> Self:
- """
- Internal version of the `take` method that sets the `_is_copy`
- attribute to keep track of the parent dataframe (using in indexing
- for the SettingWithCopyWarning).
-
- For Series this does the same as the public take (it never sets `_is_copy`).
-
- See the docstring of `take` for full explanation of the parameters.
- """
- result = self.take(indices=indices, axis=axis)
- # Maybe set copy if we didn't actually change the index.
- if self.ndim == 2 and not result._get_axis(axis).equals(self._get_axis(axis)):
- result._set_is_copy(self)
- return result
-
@final
def xs(
self,
key: IndexLabel,
axis: Axis = 0,
level: IndexLabel | None = None,
- drop_level: bool_t = True,
+ drop_level: bool = True,
) -> Self:
"""
Return cross-section from the Series/DataFrame.
@@ -4201,13 +3980,15 @@ def xs(
Examples
--------
- >>> d = {'num_legs': [4, 4, 2, 2],
- ... 'num_wings': [0, 0, 2, 2],
- ... 'class': ['mammal', 'mammal', 'mammal', 'bird'],
- ... 'animal': ['cat', 'dog', 'bat', 'penguin'],
- ... 'locomotion': ['walks', 'walks', 'flies', 'walks']}
+ >>> d = {
+ ... "num_legs": [4, 4, 2, 2],
+ ... "num_wings": [0, 0, 2, 2],
+ ... "class": ["mammal", "mammal", "mammal", "bird"],
+ ... "animal": ["cat", "dog", "bat", "penguin"],
+ ... "locomotion": ["walks", "walks", "flies", "walks"],
+ ... }
>>> df = pd.DataFrame(data=d)
- >>> df = df.set_index(['class', 'animal', 'locomotion'])
+ >>> df = df.set_index(["class", "animal", "locomotion"])
>>> df
num_legs num_wings
class animal locomotion
@@ -4218,7 +3999,7 @@ class animal locomotion
Get values at specified index
- >>> df.xs('mammal')
+ >>> df.xs("mammal")
num_legs num_wings
animal locomotion
cat walks 4 0
@@ -4227,29 +4008,28 @@ class animal locomotion
Get values at several indexes
- >>> df.xs(('mammal', 'dog', 'walks'))
+ >>> df.xs(("mammal", "dog", "walks"))
num_legs 4
num_wings 0
Name: (mammal, dog, walks), dtype: int64
Get values at specified index and level
- >>> df.xs('cat', level=1)
+ >>> df.xs("cat", level=1)
num_legs num_wings
class locomotion
mammal walks 4 0
Get values at several indexes and levels
- >>> df.xs(('bird', 'walks'),
- ... level=[0, 'locomotion'])
+ >>> df.xs(("bird", "walks"), level=[0, "locomotion"])
num_legs num_wings
animal
penguin 2 2
Get values at specified column and axis
- >>> df.xs('num_wings', axis=1)
+ >>> df.xs("num_wings", axis=1)
class animal locomotion
mammal cat walks 0
dog walks 0
@@ -4298,9 +4078,9 @@ class animal locomotion
if isinstance(loc, np.ndarray):
if loc.dtype == np.bool_:
(inds,) = loc.nonzero()
- return self._take_with_is_copy(inds, axis=axis)
+ return self.take(inds, axis=axis)
else:
- return self._take_with_is_copy(loc, axis=axis)
+ return self.take(loc, axis=axis)
if not is_scalar(loc):
new_index = index[loc]
@@ -4326,9 +4106,6 @@ class animal locomotion
result = self.iloc[loc]
result.index = new_index
- # this could be a view
- # but only in a single-dtyped view sliceable case
- result._set_is_copy(self, copy=not result._is_view)
return result
def __getitem__(self, item):
@@ -4344,9 +4121,7 @@ def _getitem_slice(self, key: slice) -> Self:
slobj = self.index._convert_slice_indexer(key, kind="getitem")
if isinstance(slobj, np.ndarray):
# reachable with DatetimeIndex
- indexer = lib.maybe_indices_to_slice(
- slobj.astype(np.intp, copy=False), len(self)
- )
+ indexer = lib.maybe_indices_to_slice(slobj.astype(np.intp), len(self))
if isinstance(indexer, np.ndarray):
# GH#43223 If we can not convert, use take
return self.take(indexer, axis=0)
@@ -4364,117 +4139,14 @@ def _slice(self, slobj: slice, axis: AxisInt = 0) -> Self:
new_mgr = self._mgr.get_slice(slobj, axis=axis)
result = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
result = result.__finalize__(self)
-
- # this could be a view
- # but only in a single-dtyped view sliceable case
- is_copy = axis != 0 or result._is_view
- result._set_is_copy(self, copy=is_copy)
return result
@final
- def _set_is_copy(self, ref: NDFrame, copy: bool_t = True) -> None:
- if not copy:
- self._is_copy = None
- else:
- assert ref is not None
- self._is_copy = weakref.ref(ref)
-
- def _check_is_chained_assignment_possible(self) -> bool_t:
- """
- Check if we are a view, have a cacher, and are of mixed type.
- If so, then force a setitem_copy check.
-
- Should be called just near setting a value
-
- Will return a boolean if it we are a view and are cached, but a
- single-dtype meaning that the cacher should be updated following
- setting.
+ def __delitem__(self, key) -> None:
"""
- if self._is_copy:
- self._check_setitem_copy(t="referent")
- return False
-
- @final
- def _check_setitem_copy(self, t: str = "setting", force: bool_t = False) -> None:
+ Delete item
"""
-
- Parameters
- ----------
- t : str, the type of setting error
- force : bool, default False
- If True, then force showing an error.
-
- validate if we are doing a setitem on a chained copy.
-
- It is technically possible to figure out that we are setting on
- a copy even WITH a multi-dtyped pandas object. In other words, some
- blocks may be views while other are not. Currently _is_view will ALWAYS
- return False for multi-blocks to avoid having to handle this case.
-
- df = DataFrame(np.arange(0,9), columns=['count'])
- df['group'] = 'b'
-
- # This technically need not raise SettingWithCopy if both are view
- # (which is not generally guaranteed but is usually True. However,
- # this is in general not a good practice and we recommend using .loc.
- df.iloc[0:5]['group'] = 'a'
-
- """
- if using_copy_on_write() or warn_copy_on_write():
- return
-
- # return early if the check is not needed
- if not (force or self._is_copy):
- return
-
- value = config.get_option("mode.chained_assignment")
- if value is None:
- return
-
- # see if the copy is not actually referred; if so, then dissolve
- # the copy weakref
- if self._is_copy is not None and not isinstance(self._is_copy, str):
- r = self._is_copy()
- if not gc.get_referents(r) or (r is not None and r.shape == self.shape):
- self._is_copy = None
- return
-
- # a custom message
- if isinstance(self._is_copy, str):
- t = self._is_copy
-
- elif t == "referent":
- t = (
- "\n"
- "A value is trying to be set on a copy of a slice from a "
- "DataFrame\n\n"
- "See the caveats in the documentation: "
- "https://pandas.pydata.org/pandas-docs/stable/user_guide/"
- "indexing.html#returning-a-view-versus-a-copy"
- )
-
- else:
- t = (
- "\n"
- "A value is trying to be set on a copy of a slice from a "
- "DataFrame.\n"
- "Try using .loc[row_indexer,col_indexer] = value "
- "instead\n\nSee the caveats in the documentation: "
- "https://pandas.pydata.org/pandas-docs/stable/user_guide/"
- "indexing.html#returning-a-view-versus-a-copy"
- )
-
- if value == "raise":
- raise SettingWithCopyError(t)
- if value == "warn":
- warnings.warn(t, SettingWithCopyWarning, stacklevel=find_stack_level())
-
- @final
- def __delitem__(self, key) -> None:
- """
- Delete item
- """
- deleted = False
+ deleted = False
maybe_shortcut = False
if self.ndim == 2 and isinstance(self.columns, MultiIndex):
@@ -4501,17 +4173,11 @@ def __delitem__(self, key) -> None:
loc = self.axes[-1].get_loc(key)
self._mgr = self._mgr.idelete(loc)
- # delete from the caches
- try:
- del self._item_cache[key]
- except KeyError:
- pass
-
# ----------------------------------------------------------------------
# Unsorted
@final
- def _check_inplace_and_allows_duplicate_labels(self, inplace: bool_t) -> None:
+ def _check_inplace_and_allows_duplicate_labels(self, inplace: bool) -> None:
if inplace and not self.flags.allows_duplicate_labels:
raise ValueError(
"Cannot specify 'inplace=True' when "
@@ -4523,15 +4189,19 @@ def get(self, key, default=None):
"""
Get item from object for given key (ex: DataFrame column).
- Returns default value if not found.
+ Returns ``default`` value if not found.
Parameters
----------
key : object
+ Key for which item should be returned.
+ default : object, default None
+ Default value to return if key is not found.
Returns
-------
same type as items contained in object
+ Item for given key or ``default`` value, if key is not found.
Examples
--------
@@ -4560,8 +4230,8 @@ def get(self, key, default=None):
2014-02-14 22.0 medium
2014-02-15 35.0 medium
- >>> ser = df['windspeed']
- >>> ser.get('2014-02-13')
+ >>> ser = df["windspeed"]
+ >>> ser.get("2014-02-13")
'high'
If the key isn't found, the default value will be used.
@@ -4569,7 +4239,7 @@ def get(self, key, default=None):
>>> df.get(["temp_celsius", "temp_kelvin"], default="default_value")
'default_value'
- >>> ser.get('2014-02-10', '[unknown]')
+ >>> ser.get("2014-02-10", "[unknown]")
'[unknown]'
"""
try:
@@ -4577,18 +4247,24 @@ def get(self, key, default=None):
except (KeyError, ValueError, IndexError):
return default
- @final
- @property
- def _is_view(self) -> bool_t:
- """Return boolean indicating if self is view of another array"""
- return self._mgr.is_view
+ @staticmethod
+ def _check_copy_deprecation(copy):
+ if copy is not lib.no_default:
+ warnings.warn(
+ "The copy keyword is deprecated and will be removed in a future "
+ "version. Copy-on-Write is active in pandas since 3.0 which utilizes "
+ "a lazy copy mechanism that defers copies until necessary. Use "
+ ".copy() to make an eager copy if necessary.",
+ DeprecationWarning,
+ stacklevel=find_stack_level(),
+ )
@final
def reindex_like(
self,
other,
method: Literal["backfill", "bfill", "pad", "ffill", "nearest"] | None = None,
- copy: bool_t | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
limit: int | None = None,
tolerance=None,
) -> Self:
@@ -4616,7 +4292,7 @@ def reindex_like(
* backfill / bfill: use next valid observation to fill gap
* nearest: use nearest valid observations to fill gap.
- copy : bool, default True
+ copy : bool, default False
Return a new object, even if the passed indexes are the same.
.. note::
@@ -4630,6 +4306,8 @@ def reindex_like(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+
+ .. deprecated:: 3.0.0
limit : int, default None
Maximum number of consecutive labels to fill for inexact matches.
tolerance : optional
@@ -4661,14 +4339,16 @@ def reindex_like(
Examples
--------
- >>> df1 = pd.DataFrame([[24.3, 75.7, 'high'],
- ... [31, 87.8, 'high'],
- ... [22, 71.6, 'medium'],
- ... [35, 95, 'medium']],
- ... columns=['temp_celsius', 'temp_fahrenheit',
- ... 'windspeed'],
- ... index=pd.date_range(start='2014-02-12',
- ... end='2014-02-15', freq='D'))
+ >>> df1 = pd.DataFrame(
+ ... [
+ ... [24.3, 75.7, "high"],
+ ... [31, 87.8, "high"],
+ ... [22, 71.6, "medium"],
+ ... [35, 95, "medium"],
+ ... ],
+ ... columns=["temp_celsius", "temp_fahrenheit", "windspeed"],
+ ... index=pd.date_range(start="2014-02-12", end="2014-02-15", freq="D"),
+ ... )
>>> df1
temp_celsius temp_fahrenheit windspeed
@@ -4677,12 +4357,11 @@ def reindex_like(
2014-02-14 22.0 71.6 medium
2014-02-15 35.0 95.0 medium
- >>> df2 = pd.DataFrame([[28, 'low'],
- ... [30, 'low'],
- ... [35.1, 'medium']],
- ... columns=['temp_celsius', 'windspeed'],
- ... index=pd.DatetimeIndex(['2014-02-12', '2014-02-13',
- ... '2014-02-15']))
+ >>> df2 = pd.DataFrame(
+ ... [[28, "low"], [30, "low"], [35.1, "medium"]],
+ ... columns=["temp_celsius", "windspeed"],
+ ... index=pd.DatetimeIndex(["2014-02-12", "2014-02-13", "2014-02-15"]),
+ ... )
>>> df2
temp_celsius windspeed
@@ -4697,10 +4376,10 @@ def reindex_like(
2014-02-14 NaN NaN NaN
2014-02-15 35.1 NaN medium
"""
+ self._check_copy_deprecation(copy)
d = other._construct_axes_dict(
axes=self._AXIS_ORDERS,
method=method,
- copy=copy,
limit=limit,
tolerance=tolerance,
)
@@ -4718,8 +4397,7 @@ def drop(
level: Level | None = ...,
inplace: Literal[True],
errors: IgnoreRaise = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def drop(
@@ -4732,8 +4410,7 @@ def drop(
level: Level | None = ...,
inplace: Literal[False] = ...,
errors: IgnoreRaise = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def drop(
@@ -4744,10 +4421,9 @@ def drop(
index: IndexLabel | ListLike = ...,
columns: IndexLabel | ListLike = ...,
level: Level | None = ...,
- inplace: bool_t = ...,
+ inplace: bool = ...,
errors: IgnoreRaise = ...,
- ) -> Self | None:
- ...
+ ) -> Self | None: ...
def drop(
self,
@@ -4757,7 +4433,7 @@ def drop(
index: IndexLabel | ListLike = None,
columns: IndexLabel | ListLike = None,
level: Level | None = None,
- inplace: bool_t = False,
+ inplace: bool = False,
errors: IgnoreRaise = "raise",
) -> Self | None:
inplace = validate_bool_kwarg(inplace, "inplace")
@@ -4795,7 +4471,7 @@ def _drop_axis(
axis,
level=None,
errors: IgnoreRaise = "raise",
- only_slice: bool_t = False,
+ only_slice: bool = False,
) -> Self:
"""
Drop labels from specified axis. Used in the ``drop`` method
@@ -4866,7 +4542,6 @@ def _drop_axis(
indexer,
axis=bm_axis,
allow_dups=True,
- copy=None,
only_slice=only_slice,
)
result = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
@@ -4876,22 +4551,17 @@ def _drop_axis(
return result.__finalize__(self)
@final
- def _update_inplace(self, result, verify_is_copy: bool_t = True) -> None:
+ def _update_inplace(self, result) -> None:
"""
Replace self internals with result.
Parameters
----------
result : same type as self
- verify_is_copy : bool, default True
- Provide is_copy checks.
"""
# NOTE: This does *not* call __finalize__ and that's an explicit
# decision that we may revisit in the future.
- self._reset_cache()
- self._clear_item_cache()
self._mgr = result._mgr
- self._maybe_update_cacher(verify_is_copy=verify_is_copy, inplace=True)
@final
def add_prefix(self, prefix: str, axis: Axis | None = None) -> Self:
@@ -4930,14 +4600,14 @@ def add_prefix(self, prefix: str, axis: Axis | None = None) -> Self:
3 4
dtype: int64
- >>> s.add_prefix('item_')
+ >>> s.add_prefix("item_")
item_0 1
item_1 2
item_2 3
item_3 4
dtype: int64
- >>> df = pd.DataFrame({'A': [1, 2, 3, 4], 'B': [3, 4, 5, 6]})
+ >>> df = pd.DataFrame({"A": [1, 2, 3, 4], "B": [3, 4, 5, 6]})
>>> df
A B
0 1 3
@@ -4945,7 +4615,7 @@ def add_prefix(self, prefix: str, axis: Axis | None = None) -> Self:
2 3 5
3 4 6
- >>> df.add_prefix('col_')
+ >>> df.add_prefix("col_")
col_A col_B
0 1 3
1 2 4
@@ -4960,12 +4630,10 @@ def add_prefix(self, prefix: str, axis: Axis | None = None) -> Self:
mapper = {axis_name: f}
- # error: Incompatible return value type (got "Optional[Self]",
- # expected "Self")
- # error: Argument 1 to "rename" of "NDFrame" has incompatible type
- # "**Dict[str, partial[str]]"; expected "Union[str, int, None]"
# error: Keywords must be strings
- return self._rename(**mapper) # type: ignore[return-value, arg-type, misc]
+ # error: No overload variant of "_rename" of "NDFrame" matches
+ # argument type "dict[Literal['index', 'columns'], Callable[[Any], str]]"
+ return self._rename(**mapper) # type: ignore[call-overload, misc]
@final
def add_suffix(self, suffix: str, axis: Axis | None = None) -> Self:
@@ -5004,14 +4672,14 @@ def add_suffix(self, suffix: str, axis: Axis | None = None) -> Self:
3 4
dtype: int64
- >>> s.add_suffix('_item')
+ >>> s.add_suffix("_item")
0_item 1
1_item 2
2_item 3
3_item 4
dtype: int64
- >>> df = pd.DataFrame({'A': [1, 2, 3, 4], 'B': [3, 4, 5, 6]})
+ >>> df = pd.DataFrame({"A": [1, 2, 3, 4], "B": [3, 4, 5, 6]})
>>> df
A B
0 1 3
@@ -5019,7 +4687,7 @@ def add_suffix(self, suffix: str, axis: Axis | None = None) -> Self:
2 3 5
3 4 6
- >>> df.add_suffix('_col')
+ >>> df.add_suffix("_col")
A_col B_col
0 1 3
1 2 4
@@ -5033,64 +4701,59 @@ def add_suffix(self, suffix: str, axis: Axis | None = None) -> Self:
axis_name = self._get_axis_name(axis)
mapper = {axis_name: f}
- # error: Incompatible return value type (got "Optional[Self]",
- # expected "Self")
- # error: Argument 1 to "rename" of "NDFrame" has incompatible type
- # "**Dict[str, partial[str]]"; expected "Union[str, int, None]"
# error: Keywords must be strings
- return self._rename(**mapper) # type: ignore[return-value, arg-type, misc]
+ # error: No overload variant of "_rename" of "NDFrame" matches argument
+ # type "dict[Literal['index', 'columns'], Callable[[Any], str]]"
+ return self._rename(**mapper) # type: ignore[call-overload, misc]
@overload
def sort_values(
self,
*,
axis: Axis = ...,
- ascending: bool_t | Sequence[bool_t] = ...,
+ ascending: bool | Sequence[bool] = ...,
inplace: Literal[False] = ...,
kind: SortKind = ...,
na_position: NaPosition = ...,
- ignore_index: bool_t = ...,
+ ignore_index: bool = ...,
key: ValueKeyFunc = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def sort_values(
self,
*,
axis: Axis = ...,
- ascending: bool_t | Sequence[bool_t] = ...,
+ ascending: bool | Sequence[bool] = ...,
inplace: Literal[True],
kind: SortKind = ...,
na_position: NaPosition = ...,
- ignore_index: bool_t = ...,
+ ignore_index: bool = ...,
key: ValueKeyFunc = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def sort_values(
self,
*,
axis: Axis = ...,
- ascending: bool_t | Sequence[bool_t] = ...,
- inplace: bool_t = ...,
+ ascending: bool | Sequence[bool] = ...,
+ inplace: bool = ...,
kind: SortKind = ...,
na_position: NaPosition = ...,
- ignore_index: bool_t = ...,
+ ignore_index: bool = ...,
key: ValueKeyFunc = ...,
- ) -> Self | None:
- ...
+ ) -> Self | None: ...
def sort_values(
self,
*,
axis: Axis = 0,
- ascending: bool_t | Sequence[bool_t] = True,
- inplace: bool_t = False,
+ ascending: bool | Sequence[bool] = True,
+ inplace: bool = False,
kind: SortKind = "quicksort",
na_position: NaPosition = "last",
- ignore_index: bool_t = False,
+ ignore_index: bool = False,
key: ValueKeyFunc | None = None,
) -> Self | None:
"""
@@ -5136,12 +4799,14 @@ def sort_values(
Examples
--------
- >>> df = pd.DataFrame({
- ... 'col1': ['A', 'A', 'B', np.nan, 'D', 'C'],
- ... 'col2': [2, 1, 9, 8, 7, 4],
- ... 'col3': [0, 1, 9, 4, 2, 3],
- ... 'col4': ['a', 'B', 'c', 'D', 'e', 'F']
- ... })
+ >>> df = pd.DataFrame(
+ ... {
+ ... "col1": ["A", "A", "B", np.nan, "D", "C"],
+ ... "col2": [2, 1, 9, 8, 7, 4],
+ ... "col3": [0, 1, 9, 4, 2, 3],
+ ... "col4": ["a", "B", "c", "D", "e", "F"],
+ ... }
+ ... )
>>> df
col1 col2 col3 col4
0 A 2 0 a
@@ -5153,7 +4818,7 @@ def sort_values(
Sort by col1
- >>> df.sort_values(by=['col1'])
+ >>> df.sort_values(by=["col1"])
col1 col2 col3 col4
0 A 2 0 a
1 A 1 1 B
@@ -5164,7 +4829,7 @@ def sort_values(
Sort by multiple columns
- >>> df.sort_values(by=['col1', 'col2'])
+ >>> df.sort_values(by=["col1", "col2"])
col1 col2 col3 col4
1 A 1 1 B
0 A 2 0 a
@@ -5175,7 +4840,7 @@ def sort_values(
Sort Descending
- >>> df.sort_values(by='col1', ascending=False)
+ >>> df.sort_values(by="col1", ascending=False)
col1 col2 col3 col4
4 D 7 2 e
5 C 4 3 F
@@ -5186,7 +4851,7 @@ def sort_values(
Putting NAs first
- >>> df.sort_values(by='col1', ascending=False, na_position='first')
+ >>> df.sort_values(by="col1", ascending=False, na_position="first")
col1 col2 col3 col4
3 NaN 8 4 D
4 D 7 2 e
@@ -5197,7 +4862,7 @@ def sort_values(
Sorting with a key function
- >>> df.sort_values(by='col4', key=lambda col: col.str.lower())
+ >>> df.sort_values(by="col4", key=lambda col: col.str.lower())
col1 col2 col3 col4
0 A 2 0 a
1 A 1 1 B
@@ -5209,10 +4874,12 @@ def sort_values(
Natural sort with the key argument,
using the `natsort ` package.
- >>> df = pd.DataFrame({
- ... "time": ['0hr', '128hr', '72hr', '48hr', '96hr'],
- ... "value": [10, 20, 30, 40, 50]
- ... })
+ >>> df = pd.DataFrame(
+ ... {
+ ... "time": ["0hr", "128hr", "72hr", "48hr", "96hr"],
+ ... "value": [10, 20, 30, 40, 50],
+ ... }
+ ... )
>>> df
time value
0 0hr 10
@@ -5222,8 +4889,7 @@ def sort_values(
4 96hr 50
>>> from natsort import index_natsorted
>>> df.sort_values(
- ... by="time",
- ... key=lambda x: np.argsort(index_natsorted(df["time"]))
+ ... by="time", key=lambda x: np.argsort(index_natsorted(df["time"]))
... )
time value
0 0hr 10
@@ -5240,15 +4906,14 @@ def sort_index(
*,
axis: Axis = ...,
level: IndexLabel = ...,
- ascending: bool_t | Sequence[bool_t] = ...,
+ ascending: bool | Sequence[bool] = ...,
inplace: Literal[True],
kind: SortKind = ...,
na_position: NaPosition = ...,
- sort_remaining: bool_t = ...,
- ignore_index: bool_t = ...,
+ sort_remaining: bool = ...,
+ ignore_index: bool = ...,
key: IndexKeyFunc = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def sort_index(
@@ -5256,15 +4921,14 @@ def sort_index(
*,
axis: Axis = ...,
level: IndexLabel = ...,
- ascending: bool_t | Sequence[bool_t] = ...,
+ ascending: bool | Sequence[bool] = ...,
inplace: Literal[False] = ...,
kind: SortKind = ...,
na_position: NaPosition = ...,
- sort_remaining: bool_t = ...,
- ignore_index: bool_t = ...,
+ sort_remaining: bool = ...,
+ ignore_index: bool = ...,
key: IndexKeyFunc = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def sort_index(
@@ -5272,27 +4936,26 @@ def sort_index(
*,
axis: Axis = ...,
level: IndexLabel = ...,
- ascending: bool_t | Sequence[bool_t] = ...,
- inplace: bool_t = ...,
+ ascending: bool | Sequence[bool] = ...,
+ inplace: bool = ...,
kind: SortKind = ...,
na_position: NaPosition = ...,
- sort_remaining: bool_t = ...,
- ignore_index: bool_t = ...,
+ sort_remaining: bool = ...,
+ ignore_index: bool = ...,
key: IndexKeyFunc = ...,
- ) -> Self | None:
- ...
+ ) -> Self | None: ...
def sort_index(
self,
*,
axis: Axis = 0,
level: IndexLabel | None = None,
- ascending: bool_t | Sequence[bool_t] = True,
- inplace: bool_t = False,
+ ascending: bool | Sequence[bool] = True,
+ inplace: bool = False,
kind: SortKind = "quicksort",
na_position: NaPosition = "last",
- sort_remaining: bool_t = True,
- ignore_index: bool_t = False,
+ sort_remaining: bool = True,
+ ignore_index: bool = False,
key: IndexKeyFunc | None = None,
) -> Self | None:
inplace = validate_bool_kwarg(inplace, "inplace")
@@ -5309,10 +4972,13 @@ def sort_index(
if inplace:
result = self
else:
- result = self.copy(deep=None)
+ result = self.copy(deep=False)
if ignore_index:
- result.index = default_index(len(self))
+ if axis == 1:
+ result.columns = default_index(len(self.columns))
+ else:
+ result.index = default_index(len(self))
if inplace:
return None
else:
@@ -5347,7 +5013,7 @@ def reindex(
columns=None,
axis: Axis | None = None,
method: ReindexMethod | None = None,
- copy: bool_t | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
level: Level | None = None,
fill_value: Scalar | None = np.nan,
limit: int | None = None,
@@ -5374,7 +5040,7 @@ def reindex(
* backfill / bfill: Use next valid observation to fill gap.
* nearest: Use nearest valid observations to fill gap.
- copy : bool, default True
+ copy : bool, default False
Return a new object, even if the passed indexes are the same.
.. note::
@@ -5388,6 +5054,8 @@ def reindex(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+
+ .. deprecated:: 3.0.0
level : int or name
Broadcast across a level, matching Index values on the
passed MultiIndex level.
@@ -5409,7 +5077,8 @@ def reindex(
Returns
-------
- {klass} with changed index.
+ {klass}
+ {klass} with changed index.
See Also
--------
@@ -5427,12 +5096,15 @@ def reindex(
We *highly* recommend using keyword arguments to clarify your
intent.
- Create a dataframe with some fictional data.
+ Create a DataFrame with some fictional data.
- >>> index = ['Firefox', 'Chrome', 'Safari', 'IE10', 'Konqueror']
- >>> df = pd.DataFrame({{'http_status': [200, 200, 404, 404, 301],
- ... 'response_time': [0.04, 0.02, 0.07, 0.08, 1.0]}},
- ... index=index)
+ >>> index = ["Firefox", "Chrome", "Safari", "IE10", "Konqueror"]
+ >>> columns = ["http_status", "response_time"]
+ >>> df = pd.DataFrame(
+ ... [[200, 0.04], [200, 0.02], [404, 0.07], [404, 0.08], [301, 1.0]],
+ ... columns=columns,
+ ... index=index,
+ ... )
>>> df
http_status response_time
Firefox 200 0.04
@@ -5441,12 +5113,11 @@ def reindex(
IE10 404 0.08
Konqueror 301 1.00
- Create a new index and reindex the dataframe. By default
+ Create a new index and reindex the DataFrame. By default
values in the new index that do not have corresponding
- records in the dataframe are assigned ``NaN``.
+ records in the DataFrame are assigned ``NaN``.
- >>> new_index = ['Safari', 'Iceweasel', 'Comodo Dragon', 'IE10',
- ... 'Chrome']
+ >>> new_index = ["Safari", "Iceweasel", "Comodo Dragon", "IE10", "Chrome"]
>>> df.reindex(new_index)
http_status response_time
Safari 404.0 0.07
@@ -5468,7 +5139,7 @@ def reindex(
IE10 404 0.08
Chrome 200 0.02
- >>> df.reindex(new_index, fill_value='missing')
+ >>> df.reindex(new_index, fill_value="missing")
http_status response_time
Safari 404 0.07
Iceweasel missing missing
@@ -5478,7 +5149,7 @@ def reindex(
We can also reindex the columns.
- >>> df.reindex(columns=['http_status', 'user_agent'])
+ >>> df.reindex(columns=["http_status", "user_agent"])
http_status user_agent
Firefox 200 NaN
Chrome 200 NaN
@@ -5488,7 +5159,7 @@ def reindex(
Or we can use "axis-style" keyword arguments
- >>> df.reindex(['http_status', 'user_agent'], axis="columns")
+ >>> df.reindex(["http_status", "user_agent"], axis="columns")
http_status user_agent
Firefox 200 NaN
Chrome 200 NaN
@@ -5497,13 +5168,14 @@ def reindex(
Konqueror 301 NaN
To further illustrate the filling functionality in
- ``reindex``, we will create a dataframe with a
+ ``reindex``, we will create a DataFrame with a
monotonically increasing index (for example, a sequence
of dates).
- >>> date_index = pd.date_range('1/1/2010', periods=6, freq='D')
- >>> df2 = pd.DataFrame({{"prices": [100, 101, np.nan, 100, 89, 88]}},
- ... index=date_index)
+ >>> date_index = pd.date_range("1/1/2010", periods=6, freq="D")
+ >>> df2 = pd.DataFrame(
+ ... {{"prices": [100, 101, np.nan, 100, 89, 88]}}, index=date_index
+ ... )
>>> df2
prices
2010-01-01 100.0
@@ -5513,10 +5185,10 @@ def reindex(
2010-01-05 89.0
2010-01-06 88.0
- Suppose we decide to expand the dataframe to cover a wider
+ Suppose we decide to expand the DataFrame to cover a wider
date range.
- >>> date_index2 = pd.date_range('12/29/2009', periods=10, freq='D')
+ >>> date_index2 = pd.date_range("12/29/2009", periods=10, freq="D")
>>> df2.reindex(date_index2)
prices
2009-12-29 NaN
@@ -5538,7 +5210,7 @@ def reindex(
For example, to back-propagate the last valid value to fill the ``NaN``
values, pass ``bfill`` as an argument to the ``method`` keyword.
- >>> df2.reindex(date_index2, method='bfill')
+ >>> df2.reindex(date_index2, method="bfill")
prices
2009-12-29 100.0
2009-12-30 100.0
@@ -5551,17 +5223,18 @@ def reindex(
2010-01-06 88.0
2010-01-07 NaN
- Please note that the ``NaN`` value present in the original dataframe
+ Please note that the ``NaN`` value present in the original DataFrame
(at index value 2010-01-03) will not be filled by any of the
value propagation schemes. This is because filling while reindexing
- does not look at dataframe values, but only compares the original and
+ does not look at DataFrame values, but only compares the original and
desired indexes. If you do want to fill in the ``NaN`` values present
- in the original dataframe, use the ``fillna()`` method.
+ in the original DataFrame, use the ``fillna()`` method.
See the :ref:`user guide ` for more.
"""
# TODO: Decide if we care about having different examples for different
# kinds
+ self._check_copy_deprecation(copy)
if index is not None and columns is not None and labels is not None:
raise TypeError("Cannot specify all of 'labels', 'index', 'columns'.")
@@ -5588,22 +5261,20 @@ def reindex(
# if all axes that are requested to reindex are equal, then only copy
# if indicated must have index names equal here as well as values
- if copy and using_copy_on_write():
- copy = False
if all(
self._get_axis(axis_name).identical(ax)
for axis_name, ax in axes.items()
if ax is not None
):
- return self.copy(deep=copy)
+ return self.copy(deep=False)
# check if we are a multi reindex
if self._needs_reindex_multi(axes, method, level):
- return self._reindex_multi(axes, copy, fill_value)
+ return self._reindex_multi(axes, fill_value)
# perform the reindex on the axes
return self._reindex_axes(
- axes, level, limit, tolerance, method, fill_value, copy
+ axes, level, limit, tolerance, method, fill_value
).__finalize__(self, method="reindex")
@final
@@ -5615,7 +5286,6 @@ def _reindex_axes(
tolerance,
method,
fill_value: Scalar | None,
- copy: bool_t | None,
) -> Self:
"""Perform the reindex for all the axes."""
obj = self
@@ -5633,15 +5303,12 @@ def _reindex_axes(
obj = obj._reindex_with_indexers(
{axis: [new_index, indexer]},
fill_value=fill_value,
- copy=copy,
allow_dups=False,
)
- # If we've made a copy once, no need to make another one
- copy = False
return obj
- def _needs_reindex_multi(self, axes, method, level: Level | None) -> bool_t:
+ def _needs_reindex_multi(self, axes, method, level: Level | None) -> bool:
"""Check if we do need a multi reindex."""
return (
(common.count_not_none(*axes.values()) == self._AXIS_LEN)
@@ -5652,7 +5319,7 @@ def _needs_reindex_multi(self, axes, method, level: Level | None) -> bool_t:
and self._can_fast_transpose
)
- def _reindex_multi(self, axes, copy, fill_value):
+ def _reindex_multi(self, axes, fill_value):
raise AbstractMethodError(self)
@final
@@ -5660,8 +5327,7 @@ def _reindex_with_indexers(
self,
reindexers,
fill_value=None,
- copy: bool_t | None = False,
- allow_dups: bool_t = False,
+ allow_dups: bool = False,
) -> Self:
"""allow_dups indicates an internal call here"""
# reindex doing multiple operations on different axes if indicated
@@ -5684,18 +5350,9 @@ def _reindex_with_indexers(
axis=baxis,
fill_value=fill_value,
allow_dups=allow_dups,
- copy=copy,
)
- # If we've made a copy once, no need to make another one
- copy = False
- if (
- (copy or copy is None)
- and new_data is self._mgr
- and not using_copy_on_write()
- ):
- new_data = new_data.copy(deep=copy)
- elif using_copy_on_write() and new_data is self._mgr:
+ if new_data is self._mgr:
new_data = new_data.copy(deep=False)
return self._constructor_from_mgr(new_data, axes=new_data.axes).__finalize__(
@@ -5710,10 +5367,11 @@ def filter(
axis: Axis | None = None,
) -> Self:
"""
- Subset the dataframe rows or columns according to the specified index labels.
+ Subset the DataFrame or Series according to the specified index labels.
- Note that this routine does not filter a dataframe on its
- contents. The filter is applied to the labels of the index.
+ For DataFrame, filter rows or columns depending on ``axis`` argument.
+ Note that this routine does not filter based on content.
+ The filter is applied to the labels of the index.
Parameters
----------
@@ -5726,11 +5384,13 @@ def filter(
axis : {0 or 'index', 1 or 'columns', None}, default None
The axis to filter on, expressed either as an index (int)
or axis name (str). By default this is the info axis, 'columns' for
- DataFrame. For `Series` this parameter is unused and defaults to `None`.
+ ``DataFrame``. For ``Series`` this parameter is unused and defaults to
+ ``None``.
Returns
-------
- same type as input object
+ Same type as caller
+ The filtered subset of the DataFrame or Series.
See Also
--------
@@ -5747,28 +5407,30 @@ def filter(
Examples
--------
- >>> df = pd.DataFrame(np.array(([1, 2, 3], [4, 5, 6])),
- ... index=['mouse', 'rabbit'],
- ... columns=['one', 'two', 'three'])
+ >>> df = pd.DataFrame(
+ ... np.array(([1, 2, 3], [4, 5, 6])),
+ ... index=["mouse", "rabbit"],
+ ... columns=["one", "two", "three"],
+ ... )
>>> df
one two three
mouse 1 2 3
rabbit 4 5 6
>>> # select columns by name
- >>> df.filter(items=['one', 'three'])
+ >>> df.filter(items=["one", "three"])
one three
mouse 1 3
rabbit 4 6
>>> # select columns by regular expression
- >>> df.filter(regex='e$', axis=1)
+ >>> df.filter(regex="e$", axis=1)
one three
mouse 1 3
rabbit 4 6
>>> # select rows containing 'bbi'
- >>> df.filter(like='bbi', axis=0)
+ >>> df.filter(like="bbi", axis=0)
one two three
rabbit 4 5 6
"""
@@ -5793,7 +5455,7 @@ def filter(
return self.reindex(**{name: items}) # type: ignore[misc]
elif like:
- def f(x) -> bool_t:
+ def f(x) -> bool:
assert like is not None # needed for mypy
return like in ensure_str(x)
@@ -5801,7 +5463,7 @@ def f(x) -> bool_t:
return self.loc(axis=axis)[values]
elif regex:
- def f(x) -> bool_t:
+ def f(x) -> bool:
return matcher.search(ensure_str(x)) is not None
matcher = re.compile(regex)
@@ -5815,12 +5477,13 @@ def head(self, n: int = 5) -> Self:
"""
Return the first `n` rows.
- This function returns the first `n` rows for the object based
- on position. It is useful for quickly testing if your object
- has the right type of data in it.
+ This function exhibits the same behavior as ``df[:n]``, returning the
+ first ``n`` rows based on position. It is useful for quickly checking
+ if your object has the right type of data in it.
- For negative values of `n`, this function returns all rows except
- the last `|n|` rows, equivalent to ``df[:n]``.
+ When ``n`` is positive, it returns the first ``n`` rows. For ``n`` equal to 0,
+ it returns an empty object. When ``n`` is negative, it returns
+ all rows except the last ``|n|`` rows, mirroring the behavior of ``df[:n]``.
If n is larger than the number of rows, this function returns all rows.
@@ -5840,8 +5503,21 @@ def head(self, n: int = 5) -> Self:
Examples
--------
- >>> df = pd.DataFrame({'animal': ['alligator', 'bee', 'falcon', 'lion',
- ... 'monkey', 'parrot', 'shark', 'whale', 'zebra']})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "animal": [
+ ... "alligator",
+ ... "bee",
+ ... "falcon",
+ ... "lion",
+ ... "monkey",
+ ... "parrot",
+ ... "shark",
+ ... "whale",
+ ... "zebra",
+ ... ]
+ ... }
+ ... )
>>> df
animal
0 alligator
@@ -5883,9 +5559,7 @@ def head(self, n: int = 5) -> Self:
4 monkey
5 parrot
"""
- if using_copy_on_write():
- return self.iloc[:n].copy()
- return self.iloc[:n]
+ return self.iloc[:n].copy()
@final
def tail(self, n: int = 5) -> Self:
@@ -5917,8 +5591,21 @@ def tail(self, n: int = 5) -> Self:
Examples
--------
- >>> df = pd.DataFrame({'animal': ['alligator', 'bee', 'falcon', 'lion',
- ... 'monkey', 'parrot', 'shark', 'whale', 'zebra']})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "animal": [
+ ... "alligator",
+ ... "bee",
+ ... "falcon",
+ ... "lion",
+ ... "monkey",
+ ... "parrot",
+ ... "shark",
+ ... "whale",
+ ... "zebra",
+ ... ]
+ ... }
+ ... )
>>> df
animal
0 alligator
@@ -5960,24 +5647,20 @@ def tail(self, n: int = 5) -> Self:
7 whale
8 zebra
"""
- if using_copy_on_write():
- if n == 0:
- return self.iloc[0:0].copy()
- return self.iloc[-n:].copy()
if n == 0:
- return self.iloc[0:0]
- return self.iloc[-n:]
+ return self.iloc[0:0].copy()
+ return self.iloc[-n:].copy()
@final
def sample(
self,
n: int | None = None,
frac: float | None = None,
- replace: bool_t = False,
+ replace: bool = False,
weights=None,
random_state: RandomState | None = None,
axis: Axis | None = None,
- ignore_index: bool_t = False,
+ ignore_index: bool = False,
) -> Self:
"""
Return a random sample of items from an axis of object.
@@ -6043,10 +5726,14 @@ def sample(
Examples
--------
- >>> df = pd.DataFrame({'num_legs': [2, 4, 8, 0],
- ... 'num_wings': [2, 0, 0, 0],
- ... 'num_specimen_seen': [10, 2, 1, 8]},
- ... index=['falcon', 'dog', 'spider', 'fish'])
+ >>> df = pd.DataFrame(
+ ... {
+ ... "num_legs": [2, 4, 8, 0],
+ ... "num_wings": [2, 0, 0, 0],
+ ... "num_specimen_seen": [10, 2, 1, 8],
+ ... },
+ ... index=["falcon", "dog", "spider", "fish"],
+ ... )
>>> df
num_legs num_wings num_specimen_seen
falcon 2 2 10
@@ -6058,7 +5745,7 @@ def sample(
Note that we use `random_state` to ensure the reproducibility of
the examples.
- >>> df['num_legs'].sample(n=3, random_state=1)
+ >>> df["num_legs"].sample(n=3, random_state=1)
fish 0
spider 8
falcon 2
@@ -6088,7 +5775,7 @@ def sample(
Using a DataFrame column as weights. Rows with larger value in the
`num_specimen_seen` column are more likely to be sampled.
- >>> df.sample(n=2, weights='num_specimen_seen', random_state=1)
+ >>> df.sample(n=2, weights="num_specimen_seen", random_state=1)
num_legs num_wings num_specimen_seen
falcon 2 2 10
fish 0 0 8
@@ -6118,13 +5805,29 @@ def sample(
return result
+ @overload
+ def pipe(
+ self,
+ func: Callable[Concatenate[Self, P], T],
+ *args: P.args,
+ **kwargs: P.kwargs,
+ ) -> T: ...
+
+ @overload
+ def pipe(
+ self,
+ func: tuple[Callable[..., T], str],
+ *args: Any,
+ **kwargs: Any,
+ ) -> T: ...
+
@final
@doc(klass=_shared_doc_kwargs["klass"])
def pipe(
self,
- func: Callable[..., T] | tuple[Callable[..., T], str],
- *args,
- **kwargs,
+ func: Callable[Concatenate[Self, P], T] | tuple[Callable[..., T], str],
+ *args: Any,
+ **kwargs: Any,
) -> T:
r"""
Apply chainable functions that expect Series or DataFrames.
@@ -6144,7 +5847,8 @@ def pipe(
Returns
-------
- the return type of ``func``.
+ The return type of ``func``.
+ The result of applying ``func`` to the Series or DataFrame.
See Also
--------
@@ -6160,10 +5864,10 @@ def pipe(
Examples
--------
- Constructing a income DataFrame from a dictionary.
+ Constructing an income DataFrame from a dictionary.
>>> data = [[8000, 1000], [9500, np.nan], [5000, 2000]]
- >>> df = pd.DataFrame(data, columns=['Salary', 'Others'])
+ >>> df = pd.DataFrame(data, columns=["Salary", "Others"])
>>> df
Salary Others
0 8000 1000.0
@@ -6185,7 +5889,8 @@ def pipe(
>>> subtract_national_insurance(
... subtract_state_tax(subtract_federal_tax(df), rate=0.12),
... rate=0.05,
- ... rate_increase=0.02) # doctest: +SKIP
+ ... rate_increase=0.02,
+ ... ) # doctest: +SKIP
You can write
@@ -6211,9 +5916,7 @@ def pipe(
... df.pipe(subtract_federal_tax)
... .pipe(subtract_state_tax, rate=0.12)
... .pipe(
- ... (subtract_national_insurance, 'df'),
- ... rate=0.05,
- ... rate_increase=0.02
+ ... (subtract_national_insurance, "df"), rate=0.05, rate_increase=0.02
... )
... )
Salary Others
@@ -6221,9 +5924,7 @@ def pipe(
1 6997.32 NaN
2 3682.80 1473.12
"""
- if using_copy_on_write():
- return common.pipe(self.copy(deep=None), func, *args, **kwargs)
- return common.pipe(self, func, *args, **kwargs)
+ return common.pipe(self.copy(deep=False), func, *args, **kwargs)
# ----------------------------------------------------------------------
# Attribute access
@@ -6349,28 +6050,11 @@ def _dir_additions(self) -> set[str]:
# ----------------------------------------------------------------------
# Consolidation of internals
- @final
- def _protect_consolidate(self, f):
- """
- Consolidate _mgr -- if the blocks have changed, then clear the
- cache
- """
- if isinstance(self._mgr, (ArrayManager, SingleArrayManager)):
- return f()
- blocks_before = len(self._mgr.blocks)
- result = f()
- if len(self._mgr.blocks) != blocks_before:
- self._clear_item_cache()
- return result
-
@final
def _consolidate_inplace(self) -> None:
"""Consolidate data in place and return None"""
- def f() -> None:
- self._mgr = self._mgr.consolidate()
-
- self._protect_consolidate(f)
+ self._mgr = self._mgr.consolidate()
@final
def _consolidate(self):
@@ -6382,15 +6066,14 @@ def _consolidate(self):
-------
consolidated : same type as caller
"""
- f = lambda: self._mgr.consolidate()
- cons_data = self._protect_consolidate(f)
+ cons_data = self._mgr.consolidate()
return self._constructor_from_mgr(cons_data, axes=cons_data.axes).__finalize__(
self
)
@final
@property
- def _is_mixed_type(self) -> bool_t:
+ def _is_mixed_type(self) -> bool:
if self._mgr.is_single_block:
# Includes all Series cases
return False
@@ -6441,10 +6124,14 @@ def dtypes(self):
Examples
--------
- >>> df = pd.DataFrame({'float': [1.0],
- ... 'int': [1],
- ... 'datetime': [pd.Timestamp('20180310')],
- ... 'string': ['foo']})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "float": [1.0],
+ ... "int": [1],
+ ... "datetime": [pd.Timestamp("20180310")],
+ ... "string": ["foo"],
+ ... }
+ ... )
>>> df.dtypes
float float64
int int64
@@ -6457,7 +6144,10 @@ def dtypes(self):
@final
def astype(
- self, dtype, copy: bool_t | None = None, errors: IgnoreRaise = "raise"
+ self,
+ dtype,
+ copy: bool | lib.NoDefault = lib.no_default,
+ errors: IgnoreRaise = "raise",
) -> Self:
"""
Cast a pandas object to a specified dtype ``dtype``.
@@ -6470,7 +6160,7 @@ def astype(
mapping, e.g. {col: dtype, ...}, where col is a column label and dtype is
a numpy.dtype or Python type to cast one or more of the DataFrame's
columns to column-specific types.
- copy : bool, default True
+ copy : bool, default False
Return a copy when ``copy=True`` (be very careful setting
``copy=False`` as changes to values then may propagate to other
pandas objects).
@@ -6486,6 +6176,8 @@ def astype(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+
+ .. deprecated:: 3.0.0
errors : {'raise', 'ignore'}, default 'raise'
Control raising of exceptions on invalid data for provided dtype.
@@ -6495,6 +6187,7 @@ def astype(
Returns
-------
same type as caller
+ The pandas object casted to the specified ``dtype``.
See Also
--------
@@ -6515,7 +6208,7 @@ def astype(
--------
Create a DataFrame:
- >>> d = {'col1': [1, 2], 'col2': [3, 4]}
+ >>> d = {"col1": [1, 2], "col2": [3, 4]}
>>> df = pd.DataFrame(data=d)
>>> df.dtypes
col1 int64
@@ -6524,33 +6217,33 @@ def astype(
Cast all columns to int32:
- >>> df.astype('int32').dtypes
+ >>> df.astype("int32").dtypes
col1 int32
col2 int32
dtype: object
Cast col1 to int32 using a dictionary:
- >>> df.astype({'col1': 'int32'}).dtypes
+ >>> df.astype({"col1": "int32"}).dtypes
col1 int32
col2 int64
dtype: object
Create a series:
- >>> ser = pd.Series([1, 2], dtype='int32')
+ >>> ser = pd.Series([1, 2], dtype="int32")
>>> ser
0 1
1 2
dtype: int32
- >>> ser.astype('int64')
+ >>> ser.astype("int64")
0 1
1 2
dtype: int64
Convert to categorical type:
- >>> ser.astype('category')
+ >>> ser.astype("category")
0 1
1 2
dtype: category
@@ -6559,8 +6252,7 @@ def astype(
Convert to ordered categorical type with custom ordering:
>>> from pandas.api.types import CategoricalDtype
- >>> cat_dtype = CategoricalDtype(
- ... categories=[2, 1], ordered=True)
+ >>> cat_dtype = CategoricalDtype(categories=[2, 1], ordered=True)
>>> ser.astype(cat_dtype)
0 1
1 2
@@ -6569,16 +6261,14 @@ def astype(
Create a series of dates:
- >>> ser_date = pd.Series(pd.date_range('20200101', periods=3))
+ >>> ser_date = pd.Series(pd.date_range("20200101", periods=3))
>>> ser_date
0 2020-01-01
1 2020-01-02
2 2020-01-03
dtype: datetime64[ns]
"""
- if copy and using_copy_on_write():
- copy = False
-
+ self._check_copy_deprecation(copy)
if is_dict_like(dtype):
if self.ndim == 1: # i.e. Series
if len(dtype) > 1 or self.name not in dtype:
@@ -6587,7 +6277,7 @@ def astype(
"the key in Series dtype mappings."
)
new_type = dtype[self.name]
- return self.astype(new_type, copy, errors)
+ return self.astype(new_type, errors=errors)
# GH#44417 cast to Series so we can use .iat below, which will be
# robust in case we
@@ -6603,16 +6293,16 @@ def astype(
f"'{col_name}' not found in columns."
)
- dtype_ser = dtype_ser.reindex(self.columns, fill_value=None, copy=False)
+ dtype_ser = dtype_ser.reindex(self.columns, fill_value=None)
results = []
for i, (col_name, col) in enumerate(self.items()):
cdt = dtype_ser.iat[i]
if isna(cdt):
- res_col = col.copy(deep=copy)
+ res_col = col.copy(deep=False)
else:
try:
- res_col = col.astype(dtype=cdt, copy=copy, errors=errors)
+ res_col = col.astype(dtype=cdt, errors=errors)
except ValueError as ex:
ex.args = (
f"{ex}: Error while type casting for column '{col_name}'",
@@ -6626,25 +6316,23 @@ def astype(
if isinstance(dtype, ExtensionDtype) and all(
arr.dtype == dtype for arr in self._mgr.arrays
):
- return self.copy(deep=copy)
+ return self.copy(deep=False)
# GH 18099/22869: columnwise conversion to extension dtype
# GH 24704: self.items handles duplicate column names
- results = [
- ser.astype(dtype, copy=copy, errors=errors) for _, ser in self.items()
- ]
+ results = [ser.astype(dtype, errors=errors) for _, ser in self.items()]
else:
# else, only a single dtype is given
- new_data = self._mgr.astype(dtype=dtype, copy=copy, errors=errors)
+ new_data = self._mgr.astype(dtype=dtype, errors=errors)
res = self._constructor_from_mgr(new_data, axes=new_data.axes)
return res.__finalize__(self, method="astype")
# GH 33113: handle empty frame or series
if not results:
- return self.copy(deep=None)
+ return self.copy(deep=False)
# GH 19920: retain column metadata after concat
- result = concat(results, axis=1, copy=False)
+ result = concat(results, axis=1)
# GH#40810 retain subclass
# error: Incompatible types in assignment
# (expression has type "Self", variable has type "DataFrame")
@@ -6655,7 +6343,7 @@ def astype(
return cast(Self, result)
@final
- def copy(self, deep: bool_t | None = True) -> Self:
+ def copy(self, deep: bool = True) -> Self:
"""
Make a copy of this object's indices and data.
@@ -6711,8 +6399,7 @@ def copy(self, deep: bool_t | None = True) -> Self:
:ref:`gotchas ` when copying in a threading
environment.
- When ``copy_on_write`` in pandas config is set to ``True``, the
- ``copy_on_write`` config takes effect even when ``deep=False``.
+ Copy-on-Write protects shallow copies against accidental modifications.
This means that any changes to the copied data would make a new copy
of the data upon write (and vice versa). Changes made to either the
original or copied variable would not be reflected in the counterpart.
@@ -6738,12 +6425,15 @@ def copy(self, deep: bool_t | None = True) -> Self:
>>> deep = s.copy()
>>> shallow = s.copy(deep=False)
- Shallow copy shares data and index with original.
+ Shallow copy shares index with original, the data is a
+ view of the original.
>>> s is shallow
False
- >>> s.values is shallow.values and s.index is shallow.index
- True
+ >>> s.values is shallow.values
+ False
+ >>> s.index is shallow.index
+ False
Deep copy has own copy of data and index.
@@ -6752,18 +6442,17 @@ def copy(self, deep: bool_t | None = True) -> Self:
>>> s.values is deep.values or s.index is deep.index
False
- Updates to the data shared by shallow copy and original is reflected
- in both (NOTE: this will no longer be true for pandas >= 3.0);
- deep copy remains unchanged.
+ The shallow copy is protected against updating the original object
+ as well. Thus, updates will only reflect in one of both objects.
>>> s.iloc[0] = 3
>>> shallow.iloc[1] = 4
>>> s
a 3
- b 4
+ b 2
dtype: int64
>>> shallow
- a 3
+ a 1
b 4
dtype: int64
>>> deep
@@ -6786,31 +6475,14 @@ def copy(self, deep: bool_t | None = True) -> Self:
0 [10, 2]
1 [3, 4]
dtype: object
-
- **Copy-on-Write is set to true**, the shallow copy is not modified
- when the original data is changed:
-
- >>> with pd.option_context("mode.copy_on_write", True):
- ... s = pd.Series([1, 2], index=["a", "b"])
- ... copy = s.copy(deep=False)
- ... s.iloc[0] = 100
- ... s
- a 100
- b 2
- dtype: int64
- >>> copy
- a 1
- b 2
- dtype: int64
"""
data = self._mgr.copy(deep=deep)
- self._clear_item_cache()
return self._constructor_from_mgr(data, axes=data.axes).__finalize__(
self, method="copy"
)
@final
- def __copy__(self, deep: bool_t = True) -> Self:
+ def __copy__(self, deep: bool = True) -> Self:
return self.copy(deep=deep)
@final
@@ -6824,7 +6496,7 @@ def __deepcopy__(self, memo=None) -> Self:
return self.copy(deep=True)
@final
- def infer_objects(self, copy: bool_t | None = None) -> Self:
+ def infer_objects(self, copy: bool | lib.NoDefault = lib.no_default) -> Self:
"""
Attempt to infer better dtypes for object columns.
@@ -6835,7 +6507,7 @@ def infer_objects(self, copy: bool_t | None = None) -> Self:
Parameters
----------
- copy : bool, default True
+ copy : bool, default False
Whether to make a copy for non-object or non-inferable columns
or Series.
@@ -6851,6 +6523,8 @@ def infer_objects(self, copy: bool_t | None = None) -> Self:
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+ .. deprecated:: 3.0.0
+
Returns
-------
same type as input object
@@ -6880,18 +6554,19 @@ def infer_objects(self, copy: bool_t | None = None) -> Self:
A int64
dtype: object
"""
- new_mgr = self._mgr.convert(copy=copy)
+ self._check_copy_deprecation(copy)
+ new_mgr = self._mgr.convert()
res = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
return res.__finalize__(self, method="infer_objects")
@final
def convert_dtypes(
self,
- infer_objects: bool_t = True,
- convert_string: bool_t = True,
- convert_integer: bool_t = True,
- convert_boolean: bool_t = True,
- convert_floating: bool_t = True,
+ infer_objects: bool = True,
+ convert_string: bool = True,
+ convert_integer: bool = True,
+ convert_boolean: bool = True,
+ convert_floating: bool = True,
dtype_backend: DtypeBackend = "numpy_nullable",
) -> Self:
"""
@@ -7023,7 +6698,7 @@ def convert_dtypes(
dtype: string
"""
check_dtype_backend(dtype_backend)
- new_mgr = self._mgr.convert_dtypes( # type: ignore[union-attr]
+ new_mgr = self._mgr.convert_dtypes(
infer_objects=infer_objects,
convert_string=convert_string,
convert_integer=convert_integer,
@@ -7037,44 +6712,25 @@ def convert_dtypes(
# ----------------------------------------------------------------------
# Filling NA's
- def _deprecate_downcast(self, downcast, method_name: str):
- # GH#40988
- if downcast is not lib.no_default:
- warnings.warn(
- f"The 'downcast' keyword in {method_name} is deprecated and "
- "will be removed in a future version. Use "
- "res.infer_objects(copy=False) to infer non-object dtype, or "
- "pd.to_numeric with the 'downcast' keyword to downcast numeric "
- "results.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- downcast = None
- return downcast
-
@final
def _pad_or_backfill(
self,
method: Literal["ffill", "bfill", "pad", "backfill"],
*,
axis: None | Axis = None,
- inplace: bool_t = False,
+ inplace: bool = False,
limit: None | int = None,
limit_area: Literal["inside", "outside"] | None = None,
- downcast: dict | None = None,
):
if axis is None:
axis = 0
axis = self._get_axis_number(axis)
method = clean_fill_method(method)
- if not self._mgr.is_single_block and axis == 1:
- # e.g. test_align_fill_method
- # TODO(3.0): once downcast is removed, we can do the .T
- # in all axis=1 cases, and remove axis kward from mgr.pad_or_backfill.
- if inplace:
+ if axis == 1:
+ if not self._mgr.is_single_block and inplace:
raise NotImplementedError()
+ # e.g. test_align_fill_method
result = self.T._pad_or_backfill(
method=method, limit=limit, limit_area=limit_area
).T
@@ -7083,11 +6739,9 @@ def _pad_or_backfill(
new_mgr = self._mgr.pad_or_backfill(
method=method,
- axis=self._get_block_manager_axis(axis),
limit=limit,
limit_area=limit_area,
inplace=inplace,
- downcast=downcast,
)
result = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
if inplace:
@@ -7100,39 +6754,30 @@ def fillna(
self,
value: Hashable | Mapping | Series | DataFrame = ...,
*,
- method: FillnaOptions | None = ...,
axis: Axis | None = ...,
inplace: Literal[False] = ...,
limit: int | None = ...,
- downcast: dict | None = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def fillna(
self,
value: Hashable | Mapping | Series | DataFrame = ...,
*,
- method: FillnaOptions | None = ...,
axis: Axis | None = ...,
inplace: Literal[True],
limit: int | None = ...,
- downcast: dict | None = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def fillna(
self,
value: Hashable | Mapping | Series | DataFrame = ...,
*,
- method: FillnaOptions | None = ...,
axis: Axis | None = ...,
- inplace: bool_t = ...,
+ inplace: bool = ...,
limit: int | None = ...,
- downcast: dict | None = ...,
- ) -> Self | None:
- ...
+ ) -> Self | None: ...
@final
@doc(
@@ -7143,14 +6788,12 @@ def fillna(
self,
value: Hashable | Mapping | Series | DataFrame | None = None,
*,
- method: FillnaOptions | None = None,
axis: Axis | None = None,
- inplace: bool_t = False,
+ inplace: bool = False,
limit: int | None = None,
- downcast: dict | None | lib.NoDefault = lib.no_default,
) -> Self | None:
"""
- Fill NA/NaN values using the specified method.
+ Fill NA/NaN values with `value`.
Parameters
----------
@@ -7160,15 +6803,6 @@ def fillna(
each index (for a Series) or column (for a DataFrame). Values not
in the dict/Series/DataFrame will not be filled. This value cannot
be a list.
- method : {{'backfill', 'bfill', 'ffill', None}}, default None
- Method to use for filling holes in reindexed Series:
-
- * ffill: propagate last valid observation forward to next valid.
- * backfill / bfill: use next valid observation to fill gap.
-
- .. deprecated:: 2.1.0
- Use ffill or bfill instead.
-
axis : {axes_single_arg}
Axis along which to fill missing values. For `Series`
this parameter is unused and defaults to 0.
@@ -7177,16 +6811,8 @@ def fillna(
other views on this object (e.g., a no-copy slice for a column in a
DataFrame).
limit : int, default None
- If method is specified, this is the maximum number of consecutive
- NaN values to forward/backward fill. In other words, if there is
- a gap with more than this number of consecutive NaNs, it will only
- be partially filled. If method is not specified, this is the
- maximum number of entries along the entire axis where NaNs will be
- filled. Must be greater than 0 if not None.
- downcast : dict, default is None
- A dict of item->dtype of what to downcast if possible,
- or the string 'infer' which will try to downcast to an appropriate
- equal type (e.g. float64 to int64 if possible).
+ This is the maximum number of entries along the entire axis
+ where NaNs will be filled. Must be greater than 0 if not None.
Returns
-------
@@ -7203,11 +6829,15 @@ def fillna(
Examples
--------
- >>> df = pd.DataFrame([[np.nan, 2, np.nan, 0],
- ... [3, 4, np.nan, 1],
- ... [np.nan, np.nan, np.nan, np.nan],
- ... [np.nan, 3, np.nan, 4]],
- ... columns=list("ABCD"))
+ >>> df = pd.DataFrame(
+ ... [
+ ... [np.nan, 2, np.nan, 0],
+ ... [3, 4, np.nan, 1],
+ ... [np.nan, np.nan, np.nan, np.nan],
+ ... [np.nan, 3, np.nan, 4],
+ ... ],
+ ... columns=list("ABCD"),
+ ... )
>>> df
A B C D
0 NaN 2.0 NaN 0.0
@@ -7259,43 +6889,20 @@ def fillna(
"""
inplace = validate_bool_kwarg(inplace, "inplace")
if inplace:
- if not PYPY and using_copy_on_write():
+ if not PYPY:
if sys.getrefcount(self) <= REF_COUNT:
warnings.warn(
_chained_assignment_method_msg,
ChainedAssignmentError,
stacklevel=2,
)
- elif (
- not PYPY
- and not using_copy_on_write()
- and self._is_view_after_cow_rules()
- ):
- ctr = sys.getrefcount(self)
- ref_count = REF_COUNT
- if isinstance(self, ABCSeries) and _check_cacher(self):
- # see https://github.com/pandas-dev/pandas/pull/56060#discussion_r1399245221
- ref_count += 1
- if ctr <= ref_count:
- warnings.warn(
- _chained_assignment_warning_method_msg,
- FutureWarning,
- stacklevel=2,
- )
- value, method = validate_fillna_kwargs(value, method)
- if method is not None:
- warnings.warn(
- f"{type(self).__name__}.fillna with 'method' is deprecated and "
- "will raise in a future version. Use obj.ffill() or obj.bfill() "
- "instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
+ if isinstance(value, (list, tuple)):
+ raise TypeError(
+ '"value" parameter must be a scalar or dict, but '
+ f'you passed a "{type(value).__name__}"'
)
- was_no_default = downcast is lib.no_default
- downcast = self._deprecate_downcast(downcast, "fillna")
-
# set the default here, so functions examining the signaure
# can detect if something was set (e.g. in groupby) (GH9221)
if axis is None:
@@ -7303,20 +6910,7 @@ def fillna(
axis = self._get_axis_number(axis)
if value is None:
- return self._pad_or_backfill(
- # error: Argument 1 to "_pad_or_backfill" of "NDFrame" has
- # incompatible type "Optional[Literal['backfill', 'bfill', 'ffill',
- # 'pad']]"; expected "Literal['ffill', 'bfill', 'pad', 'backfill']"
- method, # type: ignore[arg-type]
- axis=axis,
- limit=limit,
- inplace=inplace,
- # error: Argument "downcast" to "_fillna_with_method" of "NDFrame"
- # has incompatible type "Union[Dict[Any, Any], None,
- # Literal[_NoDefault.no_default]]"; expected
- # "Optional[Dict[Any, Any]]"
- downcast=downcast, # type: ignore[arg-type]
- )
+ raise ValueError("Must specify a fill 'value'.")
else:
if self.ndim == 1:
if isinstance(value, (dict, ABCSeries)):
@@ -7324,11 +6918,11 @@ def fillna(
# test_fillna_nonscalar
if inplace:
return None
- return self.copy(deep=None)
+ return self.copy(deep=False)
from pandas import Series
value = Series(value)
- value = value.reindex(self.index, copy=False)
+ value = value.reindex(self.index)
value = value._values
elif not is_list_like(value):
pass
@@ -7339,9 +6933,7 @@ def fillna(
f'"{type(value).__name__}"'
)
- new_data = self._mgr.fillna(
- value=value, limit=limit, inplace=inplace, downcast=downcast
- )
+ new_data = self._mgr.fillna(value=value, limit=limit, inplace=inplace)
elif isinstance(value, (dict, ABCSeries)):
if axis == 1:
@@ -7350,31 +6942,12 @@ def fillna(
"with dict/Series column "
"by column"
)
- if using_copy_on_write():
- result = self.copy(deep=None)
- else:
- result = self if inplace else self.copy()
- is_dict = isinstance(downcast, dict)
+ result = self if inplace else self.copy(deep=False)
for k, v in value.items():
if k not in result:
continue
- if was_no_default:
- downcast_k = lib.no_default
- else:
- downcast_k = (
- # error: Incompatible types in assignment (expression
- # has type "Union[Dict[Any, Any], None,
- # Literal[_NoDefault.no_default], Any]", variable has
- # type "_NoDefault")
- downcast # type: ignore[assignment]
- if not is_dict
- # error: Item "None" of "Optional[Dict[Any, Any]]" has
- # no attribute "get"
- else downcast.get(k) # type: ignore[union-attr]
- )
-
- res_k = result[k].fillna(v, limit=limit, downcast=downcast_k)
+ res_k = result[k].fillna(v, limit=limit)
if not inplace:
result[k] = res_k
@@ -7425,7 +6998,7 @@ def fillna(
new_data = result._mgr
else:
new_data = self._mgr.fillna(
- value=value, limit=limit, inplace=inplace, downcast=downcast
+ value=value, limit=limit, inplace=inplace
)
elif isinstance(value, ABCDataFrame) and self.ndim == 2:
new_data = self.where(self.notna(), value)._mgr
@@ -7446,9 +7019,7 @@ def ffill(
inplace: Literal[False] = ...,
limit: None | int = ...,
limit_area: Literal["inside", "outside"] | None = ...,
- downcast: dict | None | lib.NoDefault = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def ffill(
@@ -7458,21 +7029,17 @@ def ffill(
inplace: Literal[True],
limit: None | int = ...,
limit_area: Literal["inside", "outside"] | None = ...,
- downcast: dict | None | lib.NoDefault = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def ffill(
self,
*,
axis: None | Axis = ...,
- inplace: bool_t = ...,
+ inplace: bool = ...,
limit: None | int = ...,
limit_area: Literal["inside", "outside"] | None = ...,
- downcast: dict | None | lib.NoDefault = ...,
- ) -> Self | None:
- ...
+ ) -> Self | None: ...
@final
@doc(
@@ -7483,10 +7050,9 @@ def ffill(
self,
*,
axis: None | Axis = None,
- inplace: bool_t = False,
+ inplace: bool = False,
limit: None | int = None,
limit_area: Literal["inside", "outside"] | None = None,
- downcast: dict | None | lib.NoDefault = lib.no_default,
) -> Self | None:
"""
Fill NA/NaN values by propagating the last valid observation to next valid.
@@ -7518,11 +7084,6 @@ def ffill(
.. versionadded:: 2.2.0
- downcast : dict, default is None
- A dict of item->dtype of what to downcast if possible,
- or the string 'infer' which will try to downcast to an appropriate
- equal type (e.g. float64 to int64 if possible).
-
Returns
-------
{klass} or None
@@ -7530,11 +7091,15 @@ def ffill(
Examples
--------
- >>> df = pd.DataFrame([[np.nan, 2, np.nan, 0],
- ... [3, 4, np.nan, 1],
- ... [np.nan, np.nan, np.nan, np.nan],
- ... [np.nan, 3, np.nan, 4]],
- ... columns=list("ABCD"))
+ >>> df = pd.DataFrame(
+ ... [
+ ... [np.nan, 2, np.nan, 0],
+ ... [3, 4, np.nan, 1],
+ ... [np.nan, np.nan, np.nan, np.nan],
+ ... [np.nan, 3, np.nan, 4],
+ ... ],
+ ... columns=list("ABCD"),
+ ... )
>>> df
A B C D
0 NaN 2.0 NaN 0.0
@@ -7557,32 +7122,15 @@ def ffill(
3 3.0
dtype: float64
"""
- downcast = self._deprecate_downcast(downcast, "ffill")
inplace = validate_bool_kwarg(inplace, "inplace")
if inplace:
- if not PYPY and using_copy_on_write():
+ if not PYPY:
if sys.getrefcount(self) <= REF_COUNT:
warnings.warn(
_chained_assignment_method_msg,
ChainedAssignmentError,
stacklevel=2,
)
- elif (
- not PYPY
- and not using_copy_on_write()
- and self._is_view_after_cow_rules()
- ):
- ctr = sys.getrefcount(self)
- ref_count = REF_COUNT
- if isinstance(self, ABCSeries) and _check_cacher(self):
- # see https://github.com/pandas-dev/pandas/pull/56060#discussion_r1399245221
- ref_count += 1
- if ctr <= ref_count:
- warnings.warn(
- _chained_assignment_warning_method_msg,
- FutureWarning,
- stacklevel=2,
- )
return self._pad_or_backfill(
"ffill",
@@ -7590,45 +7138,7 @@ def ffill(
inplace=inplace,
limit=limit,
limit_area=limit_area,
- # error: Argument "downcast" to "_fillna_with_method" of "NDFrame"
- # has incompatible type "Union[Dict[Any, Any], None,
- # Literal[_NoDefault.no_default]]"; expected "Optional[Dict[Any, Any]]"
- downcast=downcast, # type: ignore[arg-type]
- )
-
- @final
- @doc(klass=_shared_doc_kwargs["klass"])
- def pad(
- self,
- *,
- axis: None | Axis = None,
- inplace: bool_t = False,
- limit: None | int = None,
- downcast: dict | None | lib.NoDefault = lib.no_default,
- ) -> Self | None:
- """
- Fill NA/NaN values by propagating the last valid observation to next valid.
-
- .. deprecated:: 2.0
-
- {klass}.pad is deprecated. Use {klass}.ffill instead.
-
- Returns
- -------
- {klass} or None
- Object with missing values filled or None if ``inplace=True``.
-
- Examples
- --------
- Please see examples for :meth:`DataFrame.ffill` or :meth:`Series.ffill`.
- """
- warnings.warn(
- "DataFrame.pad/Series.pad is deprecated. Use "
- "DataFrame.ffill/Series.ffill instead",
- FutureWarning,
- stacklevel=find_stack_level(),
)
- return self.ffill(axis=axis, inplace=inplace, limit=limit, downcast=downcast)
@overload
def bfill(
@@ -7638,9 +7148,7 @@ def bfill(
inplace: Literal[False] = ...,
limit: None | int = ...,
limit_area: Literal["inside", "outside"] | None = ...,
- downcast: dict | None | lib.NoDefault = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def bfill(
@@ -7649,21 +7157,17 @@ def bfill(
axis: None | Axis = ...,
inplace: Literal[True],
limit: None | int = ...,
- downcast: dict | None | lib.NoDefault = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def bfill(
self,
*,
axis: None | Axis = ...,
- inplace: bool_t = ...,
+ inplace: bool = ...,
limit: None | int = ...,
limit_area: Literal["inside", "outside"] | None = ...,
- downcast: dict | None | lib.NoDefault = ...,
- ) -> Self | None:
- ...
+ ) -> Self | None: ...
@final
@doc(
@@ -7674,10 +7178,9 @@ def bfill(
self,
*,
axis: None | Axis = None,
- inplace: bool_t = False,
+ inplace: bool = False,
limit: None | int = None,
limit_area: Literal["inside", "outside"] | None = None,
- downcast: dict | None | lib.NoDefault = lib.no_default,
) -> Self | None:
"""
Fill NA/NaN values by using the next valid observation to fill the gap.
@@ -7709,11 +7212,6 @@ def bfill(
.. versionadded:: 2.2.0
- downcast : dict, default is None
- A dict of item->dtype of what to downcast if possible,
- or the string 'infer' which will try to downcast to an appropriate
- equal type (e.g. float64 to int64 if possible).
-
Returns
-------
{klass} or None
@@ -7739,7 +7237,7 @@ def bfill(
With DataFrame:
- >>> df = pd.DataFrame({{'A': [1, None, None, 4], 'B': [None, 5, None, 7]}})
+ >>> df = pd.DataFrame({{"A": [1, None, None, 4], "B": [None, 5, None, 7]}})
>>> df
A B
0 1.0 NaN
@@ -7759,32 +7257,15 @@ def bfill(
2 4.0 7.0
3 4.0 7.0
"""
- downcast = self._deprecate_downcast(downcast, "bfill")
inplace = validate_bool_kwarg(inplace, "inplace")
if inplace:
- if not PYPY and using_copy_on_write():
+ if not PYPY:
if sys.getrefcount(self) <= REF_COUNT:
warnings.warn(
_chained_assignment_method_msg,
ChainedAssignmentError,
stacklevel=2,
)
- elif (
- not PYPY
- and not using_copy_on_write()
- and self._is_view_after_cow_rules()
- ):
- ctr = sys.getrefcount(self)
- ref_count = REF_COUNT
- if isinstance(self, ABCSeries) and _check_cacher(self):
- # see https://github.com/pandas-dev/pandas/pull/56060#discussion_r1399245221
- ref_count += 1
- if ctr <= ref_count:
- warnings.warn(
- _chained_assignment_warning_method_msg,
- FutureWarning,
- stacklevel=2,
- )
return self._pad_or_backfill(
"bfill",
@@ -7792,45 +7273,7 @@ def bfill(
inplace=inplace,
limit=limit,
limit_area=limit_area,
- # error: Argument "downcast" to "_fillna_with_method" of "NDFrame"
- # has incompatible type "Union[Dict[Any, Any], None,
- # Literal[_NoDefault.no_default]]"; expected "Optional[Dict[Any, Any]]"
- downcast=downcast, # type: ignore[arg-type]
- )
-
- @final
- @doc(klass=_shared_doc_kwargs["klass"])
- def backfill(
- self,
- *,
- axis: None | Axis = None,
- inplace: bool_t = False,
- limit: None | int = None,
- downcast: dict | None | lib.NoDefault = lib.no_default,
- ) -> Self | None:
- """
- Fill NA/NaN values by using the next valid observation to fill the gap.
-
- .. deprecated:: 2.0
-
- {klass}.backfill is deprecated. Use {klass}.bfill instead.
-
- Returns
- -------
- {klass} or None
- Object with missing values filled or None if ``inplace=True``.
-
- Examples
- --------
- Please see examples for :meth:`DataFrame.bfill` or :meth:`Series.bfill`.
- """
- warnings.warn(
- "DataFrame.backfill/Series.backfill is deprecated. Use "
- "DataFrame.bfill/Series.bfill instead",
- FutureWarning,
- stacklevel=find_stack_level(),
)
- return self.bfill(axis=axis, inplace=inplace, limit=limit, downcast=downcast)
@overload
def replace(
@@ -7839,11 +7282,8 @@ def replace(
value=...,
*,
inplace: Literal[False] = ...,
- limit: int | None = ...,
- regex: bool_t = ...,
- method: Literal["pad", "ffill", "bfill"] | lib.NoDefault = ...,
- ) -> Self:
- ...
+ regex: bool = ...,
+ ) -> Self: ...
@overload
def replace(
@@ -7852,11 +7292,8 @@ def replace(
value=...,
*,
inplace: Literal[True],
- limit: int | None = ...,
- regex: bool_t = ...,
- method: Literal["pad", "ffill", "bfill"] | lib.NoDefault = ...,
- ) -> None:
- ...
+ regex: bool = ...,
+ ) -> None: ...
@overload
def replace(
@@ -7864,12 +7301,9 @@ def replace(
to_replace=...,
value=...,
*,
- inplace: bool_t = ...,
- limit: int | None = ...,
- regex: bool_t = ...,
- method: Literal["pad", "ffill", "bfill"] | lib.NoDefault = ...,
- ) -> Self | None:
- ...
+ inplace: bool = ...,
+ regex: bool = ...,
+ ) -> Self | None: ...
@final
@doc(
@@ -7882,40 +7316,17 @@ def replace(
to_replace=None,
value=lib.no_default,
*,
- inplace: bool_t = False,
- limit: int | None = None,
- regex: bool_t = False,
- method: Literal["pad", "ffill", "bfill"] | lib.NoDefault = lib.no_default,
+ inplace: bool = False,
+ regex: bool = False,
) -> Self | None:
- if method is not lib.no_default:
+ if value is lib.no_default and not is_dict_like(to_replace) and regex is False:
+ # case that goes through _replace_single and defaults to method="pad"
warnings.warn(
# GH#33302
- f"The 'method' keyword in {type(self).__name__}.replace is "
- "deprecated and will be removed in a future version.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- elif limit is not None:
- warnings.warn(
- # GH#33302
- f"The 'limit' keyword in {type(self).__name__}.replace is "
- "deprecated and will be removed in a future version.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- if (
- value is lib.no_default
- and method is lib.no_default
- and not is_dict_like(to_replace)
- and regex is False
- ):
- # case that goes through _replace_single and defaults to method="pad"
- warnings.warn(
- # GH#33302
- f"{type(self).__name__}.replace without 'value' and with "
- "non-dict-like 'to_replace' is deprecated "
- "and will raise in a future version. "
- "Explicitly specify the new values instead.",
+ f"{type(self).__name__}.replace without 'value' and with "
+ "non-dict-like 'to_replace' is deprecated "
+ "and will raise in a future version. "
+ "Explicitly specify the new values instead.",
FutureWarning,
stacklevel=find_stack_level(),
)
@@ -7928,50 +7339,27 @@ def replace(
raise TypeError(
"Expecting 'to_replace' to be either a scalar, array-like, "
"dict or None, got invalid type "
- f"{repr(type(to_replace).__name__)}"
+ f"{type(to_replace).__name__!r}"
)
inplace = validate_bool_kwarg(inplace, "inplace")
if inplace:
- if not PYPY and using_copy_on_write():
+ if not PYPY:
if sys.getrefcount(self) <= REF_COUNT:
warnings.warn(
_chained_assignment_method_msg,
ChainedAssignmentError,
stacklevel=2,
)
- elif (
- not PYPY
- and not using_copy_on_write()
- and self._is_view_after_cow_rules()
- ):
- ctr = sys.getrefcount(self)
- ref_count = REF_COUNT
- if isinstance(self, ABCSeries) and _check_cacher(self):
- # in non-CoW mode, chained Series access will populate the
- # `_item_cache` which results in an increased ref count not below
- # the threshold, while we still need to warn. We detect this case
- # of a Series derived from a DataFrame through the presence of
- # checking the `_cacher`
- ref_count += 1
- if ctr <= ref_count:
- warnings.warn(
- _chained_assignment_warning_method_msg,
- FutureWarning,
- stacklevel=2,
- )
if not is_bool(regex) and to_replace is not None:
raise ValueError("'to_replace' must be 'None' if 'regex' is not a bool")
- if value is lib.no_default or method is not lib.no_default:
+ if value is lib.no_default:
# GH#36984 if the user explicitly passes value=None we want to
# respect that. We have the corner case where the user explicitly
# passes value=None *and* a method, which we interpret as meaning
# they want the (documented) default behavior.
- if method is lib.no_default:
- # TODO: get this to show up as the default in the docs?
- method = "pad"
# passing a single value that is scalar like
# when value is None (GH5319), for compat
@@ -7985,12 +7373,12 @@ def replace(
result = self.apply(
Series._replace_single,
- args=(to_replace, method, inplace, limit),
+ args=(to_replace, inplace),
)
if inplace:
return None
return result
- return self._replace_single(to_replace, method, inplace, limit)
+ return self._replace_single(to_replace, inplace)
if not is_dict_like(to_replace):
if not is_dict_like(regex):
@@ -8006,8 +7394,6 @@ def replace(
if items:
keys, values = zip(*items)
else:
- # error: Incompatible types in assignment (expression has type
- # "list[Never]", variable has type "tuple[Any, ...]")
keys, values = ([], []) # type: ignore[assignment]
are_mappings = [is_dict_like(v) for v in values]
@@ -8037,15 +7423,13 @@ def replace(
else:
to_replace, value = keys, values
- return self.replace(
- to_replace, value, inplace=inplace, limit=limit, regex=regex
- )
+ return self.replace(to_replace, value, inplace=inplace, regex=regex)
else:
# need a non-zero len on all axes
if not self.size:
if inplace:
return None
- return self.copy(deep=None)
+ return self.copy(deep=False)
if is_dict_like(to_replace):
if is_dict_like(value): # {'A' : NA} -> {'A' : 0}
@@ -8101,11 +7485,9 @@ def replace(
raise TypeError(
f"'regex' must be a string or a compiled regular expression "
f"or a list or dict of strings or regular expressions, "
- f"you passed a {repr(type(regex).__name__)}"
+ f"you passed a {type(regex).__name__!r}"
)
- return self.replace(
- regex, value, inplace=inplace, limit=limit, regex=True
- )
+ return self.replace(regex, value, inplace=inplace, regex=True)
else:
# dest iterable dict-like
if is_dict_like(value): # NA -> {'A' : 0, 'B' : -1}
@@ -8132,7 +7514,7 @@ def replace(
)
else:
raise TypeError(
- f'Invalid "to_replace" type: {repr(type(to_replace).__name__)}'
+ f'Invalid "to_replace" type: {type(to_replace).__name__!r}'
)
result = self._constructor_from_mgr(new_data, axes=new_data.axes)
@@ -8151,10 +7533,8 @@ def interpolate(
inplace: Literal[False] = ...,
limit_direction: Literal["forward", "backward", "both"] | None = ...,
limit_area: Literal["inside", "outside"] | None = ...,
- downcast: Literal["infer"] | None | lib.NoDefault = ...,
**kwargs,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def interpolate(
@@ -8166,10 +7546,8 @@ def interpolate(
inplace: Literal[True],
limit_direction: Literal["forward", "backward", "both"] | None = ...,
limit_area: Literal["inside", "outside"] | None = ...,
- downcast: Literal["infer"] | None | lib.NoDefault = ...,
**kwargs,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def interpolate(
@@ -8178,13 +7556,11 @@ def interpolate(
*,
axis: Axis = ...,
limit: int | None = ...,
- inplace: bool_t = ...,
+ inplace: bool = ...,
limit_direction: Literal["forward", "backward", "both"] | None = ...,
limit_area: Literal["inside", "outside"] | None = ...,
- downcast: Literal["infer"] | None | lib.NoDefault = ...,
**kwargs,
- ) -> Self | None:
- ...
+ ) -> Self | None: ...
@final
def interpolate(
@@ -8193,10 +7569,9 @@ def interpolate(
*,
axis: Axis = 0,
limit: int | None = None,
- inplace: bool_t = False,
+ inplace: bool = False,
limit_direction: Literal["forward", "backward", "both"] | None = None,
limit_area: Literal["inside", "outside"] | None = None,
- downcast: Literal["infer"] | None | lib.NoDefault = lib.no_default,
**kwargs,
) -> Self | None:
"""
@@ -8215,7 +7590,6 @@ def interpolate(
* 'time': Works on daily and higher resolution data to interpolate
given length of interval.
* 'index', 'values': use the actual numerical values of the index.
- * 'pad': Fill in NaNs using existing values.
* 'nearest', 'zero', 'slinear', 'quadratic', 'cubic',
'barycentric', 'polynomial': Passed to
`scipy.interpolate.interp1d`, whereas 'spline' is passed to
@@ -8239,23 +7613,9 @@ def interpolate(
0.
inplace : bool, default False
Update the data in place if possible.
- limit_direction : {{'forward', 'backward', 'both'}}, Optional
+ limit_direction : {{'forward', 'backward', 'both'}}, optional, default 'forward'
Consecutive NaNs will be filled in this direction.
- If limit is specified:
- * If 'method' is 'pad' or 'ffill', 'limit_direction' must be 'forward'.
- * If 'method' is 'backfill' or 'bfill', 'limit_direction' must be
- 'backwards'.
-
- If 'limit' is not specified:
- * If 'method' is 'backfill' or 'bfill', the default is 'backward'
- * else the default is 'forward'
-
- raises ValueError if `limit_direction` is 'forward' or 'both' and
- method is 'backfill' or 'bfill'.
- raises ValueError if `limit_direction` is 'backward' or 'both' and
- method is 'pad' or 'ffill'.
-
limit_area : {{`None`, 'inside', 'outside'}}, default None
If limit is specified, consecutive NaNs will be filled with this
restriction.
@@ -8265,12 +7625,7 @@ def interpolate(
(interpolate).
* 'outside': Only fill NaNs outside valid values (extrapolate).
- downcast : optional, 'infer' or None, defaults to None
- Downcast dtypes if possible.
-
- .. deprecated:: 2.1.0
-
- ``**kwargs`` : optional
+ **kwargs : optional
Keyword arguments to pass on to the interpolating function.
Returns
@@ -8326,7 +7681,7 @@ def interpolate(
an ``order`` (int).
>>> s = pd.Series([0, 2, np.nan, 8])
- >>> s.interpolate(method='polynomial', order=2)
+ >>> s.interpolate(method="polynomial", order=2)
0 0.000000
1 2.000000
2 4.666667
@@ -8341,18 +7696,22 @@ def interpolate(
Note how the first entry in column 'b' remains ``NaN``, because there
is no entry before it to use for interpolation.
- >>> df = pd.DataFrame([(0.0, np.nan, -1.0, 1.0),
- ... (np.nan, 2.0, np.nan, np.nan),
- ... (2.0, 3.0, np.nan, 9.0),
- ... (np.nan, 4.0, -4.0, 16.0)],
- ... columns=list('abcd'))
+ >>> df = pd.DataFrame(
+ ... [
+ ... (0.0, np.nan, -1.0, 1.0),
+ ... (np.nan, 2.0, np.nan, np.nan),
+ ... (2.0, 3.0, np.nan, 9.0),
+ ... (np.nan, 4.0, -4.0, 16.0),
+ ... ],
+ ... columns=list("abcd"),
+ ... )
>>> df
a b c d
0 0.0 NaN -1.0 1.0
1 NaN 2.0 NaN NaN
2 2.0 3.0 NaN 9.0
3 NaN 4.0 -4.0 16.0
- >>> df.interpolate(method='linear', limit_direction='forward', axis=0)
+ >>> df.interpolate(method="linear", limit_direction="forward", axis=0)
a b c d
0 0.0 NaN -1.0 1.0
1 1.0 2.0 -2.0 5.0
@@ -8361,53 +7720,23 @@ def interpolate(
Using polynomial interpolation.
- >>> df['d'].interpolate(method='polynomial', order=2)
+ >>> df["d"].interpolate(method="polynomial", order=2)
0 1.0
1 4.0
2 9.0
3 16.0
Name: d, dtype: float64
"""
- if downcast is not lib.no_default:
- # GH#40988
- warnings.warn(
- f"The 'downcast' keyword in {type(self).__name__}.interpolate "
- "is deprecated and will be removed in a future version. "
- "Call result.infer_objects(copy=False) on the result instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- downcast = None
- if downcast is not None and downcast != "infer":
- raise ValueError("downcast must be either None or 'infer'")
-
inplace = validate_bool_kwarg(inplace, "inplace")
if inplace:
- if not PYPY and using_copy_on_write():
+ if not PYPY:
if sys.getrefcount(self) <= REF_COUNT:
warnings.warn(
_chained_assignment_method_msg,
ChainedAssignmentError,
stacklevel=2,
)
- elif (
- not PYPY
- and not using_copy_on_write()
- and self._is_view_after_cow_rules()
- ):
- ctr = sys.getrefcount(self)
- ref_count = REF_COUNT
- if isinstance(self, ABCSeries) and _check_cacher(self):
- # see https://github.com/pandas-dev/pandas/pull/56060#discussion_r1399245221
- ref_count += 1
- if ctr <= ref_count:
- warnings.warn(
- _chained_assignment_warning_method_msg,
- FutureWarning,
- stacklevel=2,
- )
axis = self._get_axis_number(axis)
@@ -8419,36 +7748,11 @@ def interpolate(
if not isinstance(method, str):
raise ValueError("'method' should be a string, not None.")
- fillna_methods = ["ffill", "bfill", "pad", "backfill"]
- if method.lower() in fillna_methods:
- # GH#53581
- warnings.warn(
- f"{type(self).__name__}.interpolate with method={method} is "
- "deprecated and will raise in a future version. "
- "Use obj.ffill() or obj.bfill() instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- obj, should_transpose = self, False
- else:
- obj, should_transpose = (self.T, True) if axis == 1 else (self, False)
- if np.any(obj.dtypes == object):
- # GH#53631
- if not (obj.ndim == 2 and np.all(obj.dtypes == object)):
- # don't warn in cases that already raise
- warnings.warn(
- f"{type(self).__name__}.interpolate with object dtype is "
- "deprecated and will raise in a future version. Call "
- "obj.infer_objects(copy=False) before interpolating instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- if method in fillna_methods and "fill_value" in kwargs:
- raise ValueError(
- "'fill_value' is not a valid keyword for "
- f"{type(self).__name__}.interpolate with method from "
- f"{fillna_methods}"
+ obj, should_transpose = (self.T, True) if axis == 1 else (self, False)
+ # GH#53631
+ if np.any(obj.dtypes == object):
+ raise TypeError(
+ f"{type(self).__name__} cannot interpolate with object dtype."
)
if isinstance(obj.index, MultiIndex) and method != "linear":
@@ -8458,43 +7762,16 @@ def interpolate(
limit_direction = missing.infer_limit_direction(limit_direction, method)
- if obj.ndim == 2 and np.all(obj.dtypes == object):
- raise TypeError(
- "Cannot interpolate with all object-dtype columns "
- "in the DataFrame. Try setting at least one "
- "column to a numeric dtype."
- )
-
- if method.lower() in fillna_methods:
- # TODO(3.0): remove this case
- # TODO: warn/raise on limit_direction or kwargs which are ignored?
- # as of 2023-06-26 no tests get here with either
- if not self._mgr.is_single_block and axis == 1:
- # GH#53898
- if inplace:
- raise NotImplementedError()
- obj, axis, should_transpose = self.T, 1 - axis, True
-
- new_data = obj._mgr.pad_or_backfill(
- method=method,
- axis=self._get_block_manager_axis(axis),
- limit=limit,
- limit_area=limit_area,
- inplace=inplace,
- downcast=downcast,
- )
- else:
- index = missing.get_interp_index(method, obj.index)
- new_data = obj._mgr.interpolate(
- method=method,
- index=index,
- limit=limit,
- limit_direction=limit_direction,
- limit_area=limit_area,
- inplace=inplace,
- downcast=downcast,
- **kwargs,
- )
+ index = missing.get_interp_index(method, obj.index)
+ new_data = obj._mgr.interpolate(
+ method=method,
+ index=index,
+ limit=limit,
+ limit_direction=limit_direction,
+ limit_area=limit_area,
+ inplace=inplace,
+ **kwargs,
+ )
result = self._constructor_from_mgr(new_data, axes=new_data.axes)
if should_transpose:
@@ -8580,24 +7857,32 @@ def asof(self, where, subset=None):
Take all columns into consideration
- >>> df = pd.DataFrame({'a': [10., 20., 30., 40., 50.],
- ... 'b': [None, None, None, None, 500]},
- ... index=pd.DatetimeIndex(['2018-02-27 09:01:00',
- ... '2018-02-27 09:02:00',
- ... '2018-02-27 09:03:00',
- ... '2018-02-27 09:04:00',
- ... '2018-02-27 09:05:00']))
- >>> df.asof(pd.DatetimeIndex(['2018-02-27 09:03:30',
- ... '2018-02-27 09:04:30']))
+ >>> df = pd.DataFrame(
+ ... {
+ ... "a": [10.0, 20.0, 30.0, 40.0, 50.0],
+ ... "b": [None, None, None, None, 500],
+ ... },
+ ... index=pd.DatetimeIndex(
+ ... [
+ ... "2018-02-27 09:01:00",
+ ... "2018-02-27 09:02:00",
+ ... "2018-02-27 09:03:00",
+ ... "2018-02-27 09:04:00",
+ ... "2018-02-27 09:05:00",
+ ... ]
+ ... ),
+ ... )
+ >>> df.asof(pd.DatetimeIndex(["2018-02-27 09:03:30", "2018-02-27 09:04:30"]))
a b
2018-02-27 09:03:30 NaN NaN
2018-02-27 09:04:30 NaN NaN
Take a single column into consideration
- >>> df.asof(pd.DatetimeIndex(['2018-02-27 09:03:30',
- ... '2018-02-27 09:04:30']),
- ... subset=['a'])
+ >>> df.asof(
+ ... pd.DatetimeIndex(["2018-02-27 09:03:30", "2018-02-27 09:04:30"]),
+ ... subset=["a"],
+ ... )
a b
2018-02-27 09:03:30 30.0 NaN
2018-02-27 09:04:30 40.0 NaN
@@ -8688,8 +7973,7 @@ def isna(self) -> Self:
NA values, such as None or :attr:`numpy.NaN`, gets mapped to True
values.
Everything else gets mapped to False values. Characters such as empty
- strings ``''`` or :attr:`numpy.inf` are not considered NA values
- (unless you set ``pandas.options.mode.use_inf_as_na = True``).
+ strings ``''`` or :attr:`numpy.inf` are not considered NA values.
Returns
-------
@@ -8708,11 +7992,18 @@ def isna(self) -> Self:
--------
Show which entries in a DataFrame are NA.
- >>> df = pd.DataFrame(dict(age=[5, 6, np.nan],
- ... born=[pd.NaT, pd.Timestamp('1939-05-27'),
- ... pd.Timestamp('1940-04-25')],
- ... name=['Alfred', 'Batman', ''],
- ... toy=[None, 'Batmobile', 'Joker']))
+ >>> df = pd.DataFrame(
+ ... dict(
+ ... age=[5, 6, np.nan],
+ ... born=[
+ ... pd.NaT,
+ ... pd.Timestamp("1939-05-27"),
+ ... pd.Timestamp("1940-04-25"),
+ ... ],
+ ... name=["Alfred", "Batman", ""],
+ ... toy=[None, "Batmobile", "Joker"],
+ ... )
+ ... )
>>> df
age born name toy
0 5.0 NaT Alfred None
@@ -8753,8 +8044,7 @@ def notna(self) -> Self:
Return a boolean same-sized object indicating if the values are not NA.
Non-missing values get mapped to True. Characters such as empty
- strings ``''`` or :attr:`numpy.inf` are not considered NA values
- (unless you set ``pandas.options.mode.use_inf_as_na = True``).
+ strings ``''`` or :attr:`numpy.inf` are not considered NA values.
NA values, such as None or :attr:`numpy.NaN`, get mapped to False
values.
@@ -8775,11 +8065,18 @@ def notna(self) -> Self:
--------
Show which entries in a DataFrame are not NA.
- >>> df = pd.DataFrame(dict(age=[5, 6, np.nan],
- ... born=[pd.NaT, pd.Timestamp('1939-05-27'),
- ... pd.Timestamp('1940-04-25')],
- ... name=['Alfred', 'Batman', ''],
- ... toy=[None, 'Batmobile', 'Joker']))
+ >>> df = pd.DataFrame(
+ ... dict(
+ ... age=[5, 6, np.nan],
+ ... born=[
+ ... pd.NaT,
+ ... pd.Timestamp("1939-05-27"),
+ ... pd.Timestamp("1940-04-25"),
+ ... ],
+ ... name=["Alfred", "Batman", ""],
+ ... toy=[None, "Batmobile", "Joker"],
+ ... )
+ ... )
>>> df
age born name toy
0 5.0 NaT Alfred None
@@ -8814,7 +8111,7 @@ def notnull(self) -> Self:
return notna(self).__finalize__(self, method="notnull")
@final
- def _clip_with_scalar(self, lower, upper, inplace: bool_t = False):
+ def _clip_with_scalar(self, lower, upper, inplace: bool = False):
if (lower is not None and np.any(isna(lower))) or (
upper is not None and np.any(isna(upper))
):
@@ -8825,15 +8122,11 @@ def _clip_with_scalar(self, lower, upper, inplace: bool_t = False):
if lower is not None:
cond = mask | (self >= lower)
- result = result.where(
- cond, lower, inplace=inplace
- ) # type: ignore[assignment]
+ result = result.where(cond, lower, inplace=inplace) # type: ignore[assignment]
if upper is not None:
cond = mask | (self <= upper)
result = self if inplace else result
- result = result.where(
- cond, upper, inplace=inplace
- ) # type: ignore[assignment]
+ result = result.where(cond, upper, inplace=inplace) # type: ignore[assignment]
return result
@@ -8879,8 +8172,7 @@ def clip(
axis: Axis | None = ...,
inplace: Literal[False] = ...,
**kwargs,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def clip(
@@ -8891,8 +8183,7 @@ def clip(
axis: Axis | None = ...,
inplace: Literal[True],
**kwargs,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def clip(
@@ -8901,10 +8192,9 @@ def clip(
upper=...,
*,
axis: Axis | None = ...,
- inplace: bool_t = ...,
+ inplace: bool = ...,
**kwargs,
- ) -> Self | None:
- ...
+ ) -> Self | None: ...
@final
def clip(
@@ -8913,7 +8203,7 @@ def clip(
upper=None,
*,
axis: Axis | None = None,
- inplace: bool_t = False,
+ inplace: bool = False,
**kwargs,
) -> Self | None:
"""
@@ -8938,7 +8228,7 @@ def clip(
For `Series` this parameter is unused and defaults to `None`.
inplace : bool, default False
Whether to perform the operation in place on the data.
- *args, **kwargs
+ **kwargs
Additional keywords have no effect but might be accepted
for compatibility with numpy.
@@ -8951,12 +8241,12 @@ def clip(
See Also
--------
Series.clip : Trim values at input threshold in series.
- DataFrame.clip : Trim values at input threshold in dataframe.
+ DataFrame.clip : Trim values at input threshold in DataFrame.
numpy.clip : Clip (limit) the values in an array.
Examples
--------
- >>> data = {'col_0': [9, -3, 0, -1, 5], 'col_1': [-2, -7, 6, 8, -5]}
+ >>> data = {"col_0": [9, -3, 0, -1, 5], "col_1": [-2, -7, 6, 8, -5]}
>>> df = pd.DataFrame(data)
>>> df
col_0 col_1
@@ -9018,38 +8308,22 @@ def clip(
>>> df.clip(t, axis=0)
col_0 col_1
- 0 9 2
- 1 -3 -4
- 2 0 6
- 3 6 8
- 4 5 3
+ 0 9.0 2.0
+ 1 -3.0 -4.0
+ 2 0.0 6.0
+ 3 6.0 8.0
+ 4 5.0 3.0
"""
inplace = validate_bool_kwarg(inplace, "inplace")
if inplace:
- if not PYPY and using_copy_on_write():
+ if not PYPY:
if sys.getrefcount(self) <= REF_COUNT:
warnings.warn(
_chained_assignment_method_msg,
ChainedAssignmentError,
stacklevel=2,
)
- elif (
- not PYPY
- and not using_copy_on_write()
- and self._is_view_after_cow_rules()
- ):
- ctr = sys.getrefcount(self)
- ref_count = REF_COUNT
- if isinstance(self, ABCSeries) and hasattr(self, "_cacher"):
- # see https://github.com/pandas-dev/pandas/pull/56060#discussion_r1399245221
- ref_count += 1
- if ctr <= ref_count:
- warnings.warn(
- _chained_assignment_warning_method_msg,
- FutureWarning,
- stacklevel=2,
- )
axis = nv.validate_clip_with_axis(axis, (), kwargs)
if axis is not None:
@@ -9107,7 +8381,7 @@ def asfreq(
freq: Frequency,
method: FillnaOptions | None = None,
how: Literal["start", "end"] | None = None,
- normalize: bool_t = False,
+ normalize: bool = False,
fill_value: Hashable | None = None,
) -> Self:
"""
@@ -9162,16 +8436,16 @@ def asfreq(
Notes
-----
- To learn more about the frequency strings, please see `this link
- `__.
+ To learn more about the frequency strings, please see
+ :ref:`this link`.
Examples
--------
Start by creating a series with 4 one minute timestamps.
- >>> index = pd.date_range('1/1/2000', periods=4, freq='min')
+ >>> index = pd.date_range("1/1/2000", periods=4, freq="min")
>>> series = pd.Series([0.0, None, 2.0, 3.0], index=index)
- >>> df = pd.DataFrame({{'s': series}})
+ >>> df = pd.DataFrame({{"s": series}})
>>> df
s
2000-01-01 00:00:00 0.0
@@ -9181,7 +8455,7 @@ def asfreq(
Upsample the series into 30 second bins.
- >>> df.asfreq(freq='30s')
+ >>> df.asfreq(freq="30s")
s
2000-01-01 00:00:00 0.0
2000-01-01 00:00:30 NaN
@@ -9193,7 +8467,7 @@ def asfreq(
Upsample again, providing a ``fill value``.
- >>> df.asfreq(freq='30s', fill_value=9.0)
+ >>> df.asfreq(freq="30s", fill_value=9.0)
s
2000-01-01 00:00:00 0.0
2000-01-01 00:00:30 9.0
@@ -9205,7 +8479,7 @@ def asfreq(
Upsample again, providing a ``method``.
- >>> df.asfreq(freq='30s', method='bfill')
+ >>> df.asfreq(freq="30s", method="bfill")
s
2000-01-01 00:00:00 0.0
2000-01-01 00:00:30 NaN
@@ -9227,7 +8501,7 @@ def asfreq(
)
@final
- def at_time(self, time, asof: bool_t = False, axis: Axis | None = None) -> Self:
+ def at_time(self, time, asof: bool = False, axis: Axis | None = None) -> Self:
"""
Select values at particular time of day (e.g., 9:30AM).
@@ -9241,6 +8515,7 @@ def at_time(self, time, asof: bool_t = False, axis: Axis | None = None) -> Self:
Returns
-------
Series or DataFrame
+ The values with the specified time.
Raises
------
@@ -9257,8 +8532,8 @@ def at_time(self, time, asof: bool_t = False, axis: Axis | None = None) -> Self:
Examples
--------
- >>> i = pd.date_range('2018-04-09', periods=4, freq='12h')
- >>> ts = pd.DataFrame({'A': [1, 2, 3, 4]}, index=i)
+ >>> i = pd.date_range("2018-04-09", periods=4, freq="12h")
+ >>> ts = pd.DataFrame({"A": [1, 2, 3, 4]}, index=i)
>>> ts
A
2018-04-09 00:00:00 1
@@ -9266,7 +8541,7 @@ def at_time(self, time, asof: bool_t = False, axis: Axis | None = None) -> Self:
2018-04-10 00:00:00 3
2018-04-10 12:00:00 4
- >>> ts.at_time('12:00')
+ >>> ts.at_time("12:00")
A
2018-04-09 12:00:00 2
2018-04-10 12:00:00 4
@@ -9281,7 +8556,7 @@ def at_time(self, time, asof: bool_t = False, axis: Axis | None = None) -> Self:
raise TypeError("Index must be DatetimeIndex")
indexer = index.indexer_at_time(time, asof=asof)
- return self._take_with_is_copy(indexer, axis=axis)
+ return self.take(indexer, axis=axis)
@final
def between_time(
@@ -9329,8 +8604,8 @@ def between_time(
Examples
--------
- >>> i = pd.date_range('2018-04-09', periods=4, freq='1D20min')
- >>> ts = pd.DataFrame({'A': [1, 2, 3, 4]}, index=i)
+ >>> i = pd.date_range("2018-04-09", periods=4, freq="1D20min")
+ >>> ts = pd.DataFrame({"A": [1, 2, 3, 4]}, index=i)
>>> ts
A
2018-04-09 00:00:00 1
@@ -9338,7 +8613,7 @@ def between_time(
2018-04-11 00:40:00 3
2018-04-12 01:00:00 4
- >>> ts.between_time('0:15', '0:45')
+ >>> ts.between_time("0:15", "0:45")
A
2018-04-10 00:20:00 2
2018-04-11 00:40:00 3
@@ -9346,7 +8621,7 @@ def between_time(
You get the times that are *not* between two times by setting
``start_time`` later than ``end_time``:
- >>> ts.between_time('0:45', '0:15')
+ >>> ts.between_time("0:45", "0:15")
A
2018-04-09 00:00:00 1
2018-04-12 01:00:00 4
@@ -9366,14 +8641,13 @@ def between_time(
include_start=left_inclusive,
include_end=right_inclusive,
)
- return self._take_with_is_copy(indexer, axis=axis)
+ return self.take(indexer, axis=axis)
@final
@doc(klass=_shared_doc_kwargs["klass"])
def resample(
self,
rule,
- axis: Axis | lib.NoDefault = lib.no_default,
closed: Literal["right", "left"] | None = None,
label: Literal["right", "left"] | None = None,
convention: Literal["start", "end", "s", "e"] | lib.NoDefault = lib.no_default,
@@ -9382,7 +8656,7 @@ def resample(
level: Level | None = None,
origin: str | TimestampConvertibleTypes = "start_day",
offset: TimedeltaConvertibleTypes | None = None,
- group_keys: bool_t = False,
+ group_keys: bool = False,
) -> Resampler:
"""
Resample time-series data.
@@ -9396,13 +8670,6 @@ def resample(
----------
rule : DateOffset, Timedelta or str
The offset string or object representing target conversion.
- axis : {{0 or 'index', 1 or 'columns'}}, default 0
- Which axis to use for up- or down-sampling. For `Series` this parameter
- is unused and defaults to 0. Must be
- `DatetimeIndex`, `TimedeltaIndex` or `PeriodIndex`.
-
- .. deprecated:: 2.0.0
- Use frame.T.resample(...) instead.
closed : {{'right', 'left'}}, default None
Which side of bin interval is closed. The default is 'left'
for all frequency offsets except for 'ME', 'YE', 'QE', 'BME',
@@ -9434,7 +8701,7 @@ def resample(
origin : Timestamp or str, default 'start_day'
The timestamp on which to adjust the grouping. The timezone of origin
must match the timezone of the index.
- If string, must be one of the following:
+ If string, must be Timestamp convertible or one of the following:
- 'epoch': `origin` is 1970-01-01
- 'start': `origin` is the first value of the timeseries
@@ -9491,7 +8758,7 @@ def resample(
--------
Start by creating a series with 9 one minute timestamps.
- >>> index = pd.date_range('1/1/2000', periods=9, freq='min')
+ >>> index = pd.date_range("1/1/2000", periods=9, freq="min")
>>> series = pd.Series(range(9), index=index)
>>> series
2000-01-01 00:00:00 0
@@ -9508,7 +8775,7 @@ def resample(
Downsample the series into 3 minute bins and sum the values
of the timestamps falling into a bin.
- >>> series.resample('3min').sum()
+ >>> series.resample("3min").sum()
2000-01-01 00:00:00 3
2000-01-01 00:03:00 12
2000-01-01 00:06:00 21
@@ -9522,7 +8789,7 @@ def resample(
value in the resampled bucket with the label ``2000-01-01 00:03:00``
does not include 3 (if it did, the summed value would be 6, not 3).
- >>> series.resample('3min', label='right').sum()
+ >>> series.resample("3min", label="right").sum()
2000-01-01 00:03:00 3
2000-01-01 00:06:00 12
2000-01-01 00:09:00 21
@@ -9531,7 +8798,7 @@ def resample(
To include this value close the right side of the bin interval,
as shown below.
- >>> series.resample('3min', label='right', closed='right').sum()
+ >>> series.resample("3min", label="right", closed="right").sum()
2000-01-01 00:00:00 0
2000-01-01 00:03:00 6
2000-01-01 00:06:00 15
@@ -9540,7 +8807,7 @@ def resample(
Upsample the series into 30 second bins.
- >>> series.resample('30s').asfreq()[0:5] # Select first 5 rows
+ >>> series.resample("30s").asfreq()[0:5] # Select first 5 rows
2000-01-01 00:00:00 0.0
2000-01-01 00:00:30 NaN
2000-01-01 00:01:00 1.0
@@ -9551,7 +8818,7 @@ def resample(
Upsample the series into 30 second bins and fill the ``NaN``
values using the ``ffill`` method.
- >>> series.resample('30s').ffill()[0:5]
+ >>> series.resample("30s").ffill()[0:5]
2000-01-01 00:00:00 0
2000-01-01 00:00:30 0
2000-01-01 00:01:00 1
@@ -9562,7 +8829,7 @@ def resample(
Upsample the series into 30 second bins and fill the
``NaN`` values using the ``bfill`` method.
- >>> series.resample('30s').bfill()[0:5]
+ >>> series.resample("30s").bfill()[0:5]
2000-01-01 00:00:00 0
2000-01-01 00:00:30 1
2000-01-01 00:01:00 1
@@ -9574,8 +8841,7 @@ def resample(
>>> def custom_resampler(arraylike):
... return np.sum(arraylike) + 5
- ...
- >>> series.resample('3min').apply(custom_resampler)
+ >>> series.resample("3min").apply(custom_resampler)
2000-01-01 00:00:00 8
2000-01-01 00:03:00 17
2000-01-01 00:06:00 26
@@ -9584,12 +8850,9 @@ def resample(
For DataFrame objects, the keyword `on` can be used to specify the
column instead of the index for resampling.
- >>> d = {{'price': [10, 11, 9, 13, 14, 18, 17, 19],
- ... 'volume': [50, 60, 40, 100, 50, 100, 40, 50]}}
- >>> df = pd.DataFrame(d)
- >>> df['week_starting'] = pd.date_range('01/01/2018',
- ... periods=8,
- ... freq='W')
+ >>> df = pd.DataFrame([10, 11, 9, 13, 14, 18, 17, 19], columns=["price"])
+ >>> df["volume"] = [50, 60, 40, 100, 50, 100, 40, 50]
+ >>> df["week_starting"] = pd.date_range("01/01/2018", periods=8, freq="W")
>>> df
price volume week_starting
0 10 50 2018-01-07
@@ -9600,7 +8863,7 @@ def resample(
5 18 100 2018-02-11
6 17 40 2018-02-18
7 19 50 2018-02-25
- >>> df.resample('ME', on='week_starting').mean()
+ >>> df.resample("ME", on="week_starting").mean()
price volume
week_starting
2018-01-31 10.75 62.5
@@ -9609,14 +8872,20 @@ def resample(
For a DataFrame with MultiIndex, the keyword `level` can be used to
specify on which level the resampling needs to take place.
- >>> days = pd.date_range('1/1/2000', periods=4, freq='D')
- >>> d2 = {{'price': [10, 11, 9, 13, 14, 18, 17, 19],
- ... 'volume': [50, 60, 40, 100, 50, 100, 40, 50]}}
+ >>> days = pd.date_range("1/1/2000", periods=4, freq="D")
>>> df2 = pd.DataFrame(
- ... d2,
- ... index=pd.MultiIndex.from_product(
- ... [days, ['morning', 'afternoon']]
- ... )
+ ... [
+ ... [10, 50],
+ ... [11, 60],
+ ... [9, 40],
+ ... [13, 100],
+ ... [14, 50],
+ ... [18, 100],
+ ... [17, 40],
+ ... [19, 50],
+ ... ],
+ ... columns=["price", "volume"],
+ ... index=pd.MultiIndex.from_product([days, ["morning", "afternoon"]]),
... )
>>> df2
price volume
@@ -9628,7 +8897,7 @@ def resample(
afternoon 18 100
2000-01-04 morning 17 40
afternoon 19 50
- >>> df2.resample('D', level=0).sum()
+ >>> df2.resample("D", level=0).sum()
price volume
2000-01-01 21 110
2000-01-02 22 140
@@ -9637,8 +8906,8 @@ def resample(
If you want to adjust the start of the bins based on a fixed timestamp:
- >>> start, end = '2000-10-01 23:30:00', '2000-10-02 00:30:00'
- >>> rng = pd.date_range(start, end, freq='7min')
+ >>> start, end = "2000-10-01 23:30:00", "2000-10-02 00:30:00"
+ >>> rng = pd.date_range(start, end, freq="7min")
>>> ts = pd.Series(np.arange(len(rng)) * 3, index=rng)
>>> ts
2000-10-01 23:30:00 0
@@ -9652,7 +8921,7 @@ def resample(
2000-10-02 00:26:00 24
Freq: 7min, dtype: int64
- >>> ts.resample('17min').sum()
+ >>> ts.resample("17min").sum()
2000-10-01 23:14:00 0
2000-10-01 23:31:00 9
2000-10-01 23:48:00 21
@@ -9660,7 +8929,7 @@ def resample(
2000-10-02 00:22:00 24
Freq: 17min, dtype: int64
- >>> ts.resample('17min', origin='epoch').sum()
+ >>> ts.resample("17min", origin="epoch").sum()
2000-10-01 23:18:00 0
2000-10-01 23:35:00 18
2000-10-01 23:52:00 27
@@ -9668,7 +8937,7 @@ def resample(
2000-10-02 00:26:00 24
Freq: 17min, dtype: int64
- >>> ts.resample('17min', origin='2000-01-01').sum()
+ >>> ts.resample("17min", origin="2000-01-01").sum()
2000-10-01 23:24:00 3
2000-10-01 23:41:00 15
2000-10-01 23:58:00 45
@@ -9678,14 +8947,14 @@ def resample(
If you want to adjust the start of the bins with an `offset` Timedelta, the two
following lines are equivalent:
- >>> ts.resample('17min', origin='start').sum()
+ >>> ts.resample("17min", origin="start").sum()
2000-10-01 23:30:00 9
2000-10-01 23:47:00 21
2000-10-02 00:04:00 54
2000-10-02 00:21:00 24
Freq: 17min, dtype: int64
- >>> ts.resample('17min', offset='23h30min').sum()
+ >>> ts.resample("17min", offset="23h30min").sum()
2000-10-01 23:30:00 9
2000-10-01 23:47:00 21
2000-10-02 00:04:00 54
@@ -9694,7 +8963,7 @@ def resample(
If you want to take the largest Timestamp as the end of the bins:
- >>> ts.resample('17min', origin='end').sum()
+ >>> ts.resample("17min", origin="end").sum()
2000-10-01 23:35:00 0
2000-10-01 23:52:00 18
2000-10-02 00:09:00 27
@@ -9705,7 +8974,7 @@ def resample(
midnight of the largest Timestamp as the end of the bins and drop the bins
not containing data:
- >>> ts.resample('17min', origin='end_day').sum()
+ >>> ts.resample("17min", origin="end_day").sum()
2000-10-01 23:38:00 3
2000-10-01 23:55:00 15
2000-10-02 00:12:00 45
@@ -9714,25 +8983,6 @@ def resample(
"""
from pandas.core.resample import get_resampler
- if axis is not lib.no_default:
- axis = self._get_axis_number(axis)
- if axis == 1:
- warnings.warn(
- "DataFrame.resample with axis=1 is deprecated. Do "
- "`frame.T.resample(...)` without axis instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- warnings.warn(
- f"The 'axis' keyword in {type(self).__name__}.resample is "
- "deprecated and will be removed in a future version.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- axis = 0
-
if kind is not lib.no_default:
# GH#55895
warnings.warn(
@@ -9762,7 +9012,6 @@ def resample(
freq=rule,
label=label,
closed=closed,
- axis=axis,
kind=kind,
convention=convention,
key=on,
@@ -9772,178 +9021,15 @@ def resample(
group_keys=group_keys,
)
- @final
- def first(self, offset) -> Self:
- """
- Select initial periods of time series data based on a date offset.
-
- .. deprecated:: 2.1
- :meth:`.first` is deprecated and will be removed in a future version.
- Please create a mask and filter using `.loc` instead.
-
- For a DataFrame with a sorted DatetimeIndex, this function can
- select the first few rows based on a date offset.
-
- Parameters
- ----------
- offset : str, DateOffset or dateutil.relativedelta
- The offset length of the data that will be selected. For instance,
- '1ME' will display all the rows having their index within the first month.
-
- Returns
- -------
- Series or DataFrame
- A subset of the caller.
-
- Raises
- ------
- TypeError
- If the index is not a :class:`DatetimeIndex`
-
- See Also
- --------
- last : Select final periods of time series based on a date offset.
- at_time : Select values at a particular time of the day.
- between_time : Select values between particular times of the day.
-
- Examples
- --------
- >>> i = pd.date_range('2018-04-09', periods=4, freq='2D')
- >>> ts = pd.DataFrame({'A': [1, 2, 3, 4]}, index=i)
- >>> ts
- A
- 2018-04-09 1
- 2018-04-11 2
- 2018-04-13 3
- 2018-04-15 4
-
- Get the rows for the first 3 days:
-
- >>> ts.first('3D')
- A
- 2018-04-09 1
- 2018-04-11 2
-
- Notice the data for 3 first calendar days were returned, not the first
- 3 days observed in the dataset, and therefore data for 2018-04-13 was
- not returned.
- """
- warnings.warn(
- "first is deprecated and will be removed in a future version. "
- "Please create a mask and filter using `.loc` instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- if not isinstance(self.index, DatetimeIndex):
- raise TypeError("'first' only supports a DatetimeIndex index")
-
- if len(self.index) == 0:
- return self.copy(deep=False)
-
- offset = to_offset(offset)
- if not isinstance(offset, Tick) and offset.is_on_offset(self.index[0]):
- # GH#29623 if first value is end of period, remove offset with n = 1
- # before adding the real offset
- end_date = end = self.index[0] - offset.base + offset
- else:
- end_date = end = self.index[0] + offset
-
- # Tick-like, e.g. 3 weeks
- if isinstance(offset, Tick) and end_date in self.index:
- end = self.index.searchsorted(end_date, side="left")
- return self.iloc[:end]
-
- return self.loc[:end]
-
- @final
- def last(self, offset) -> Self:
- """
- Select final periods of time series data based on a date offset.
-
- .. deprecated:: 2.1
- :meth:`.last` is deprecated and will be removed in a future version.
- Please create a mask and filter using `.loc` instead.
-
- For a DataFrame with a sorted DatetimeIndex, this function
- selects the last few rows based on a date offset.
-
- Parameters
- ----------
- offset : str, DateOffset, dateutil.relativedelta
- The offset length of the data that will be selected. For instance,
- '3D' will display all the rows having their index within the last 3 days.
-
- Returns
- -------
- Series or DataFrame
- A subset of the caller.
-
- Raises
- ------
- TypeError
- If the index is not a :class:`DatetimeIndex`
-
- See Also
- --------
- first : Select initial periods of time series based on a date offset.
- at_time : Select values at a particular time of the day.
- between_time : Select values between particular times of the day.
-
- Notes
- -----
- .. deprecated:: 2.1.0
- Please create a mask and filter using `.loc` instead
-
- Examples
- --------
- >>> i = pd.date_range('2018-04-09', periods=4, freq='2D')
- >>> ts = pd.DataFrame({'A': [1, 2, 3, 4]}, index=i)
- >>> ts
- A
- 2018-04-09 1
- 2018-04-11 2
- 2018-04-13 3
- 2018-04-15 4
-
- Get the rows for the last 3 days:
-
- >>> ts.last('3D') # doctest: +SKIP
- A
- 2018-04-13 3
- 2018-04-15 4
-
- Notice the data for 3 last calendar days were returned, not the last
- 3 observed days in the dataset, and therefore data for 2018-04-11 was
- not returned.
- """
- warnings.warn(
- "last is deprecated and will be removed in a future version. "
- "Please create a mask and filter using `.loc` instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- if not isinstance(self.index, DatetimeIndex):
- raise TypeError("'last' only supports a DatetimeIndex index")
-
- if len(self.index) == 0:
- return self.copy(deep=False)
-
- offset = to_offset(offset)
-
- start_date = self.index[-1] - offset
- start = self.index.searchsorted(start_date, side="right")
- return self.iloc[start:]
-
@final
def rank(
self,
axis: Axis = 0,
method: Literal["average", "min", "max", "first", "dense"] = "average",
- numeric_only: bool_t = False,
+ numeric_only: bool = False,
na_option: Literal["keep", "top", "bottom"] = "keep",
- ascending: bool_t = True,
- pct: bool_t = False,
+ ascending: bool = True,
+ pct: bool = False,
) -> Self:
"""
Compute numerical data ranks (1 through n) along axis.
@@ -9996,9 +9082,12 @@ def rank(
Examples
--------
- >>> df = pd.DataFrame(data={'Animal': ['cat', 'penguin', 'dog',
- ... 'spider', 'snake'],
- ... 'Number_legs': [4, 2, 4, 8, np.nan]})
+ >>> df = pd.DataFrame(
+ ... data={
+ ... "Animal": ["cat", "penguin", "dog", "spider", "snake"],
+ ... "Number_legs": [4, 2, 4, 8, np.nan],
+ ... }
+ ... )
>>> df
Animal Number_legs
0 cat 4.0
@@ -10032,10 +9121,10 @@ def rank(
* pct_rank: when setting ``pct = True``, the ranking is expressed as
percentile rank.
- >>> df['default_rank'] = df['Number_legs'].rank()
- >>> df['max_rank'] = df['Number_legs'].rank(method='max')
- >>> df['NA_bottom'] = df['Number_legs'].rank(na_option='bottom')
- >>> df['pct_rank'] = df['Number_legs'].rank(pct=True)
+ >>> df["default_rank"] = df["Number_legs"].rank()
+ >>> df["max_rank"] = df["Number_legs"].rank(method="max")
+ >>> df["NA_bottom"] = df["Number_legs"].rank(na_option="bottom")
+ >>> df["pct_rank"] = df["Number_legs"].rank(pct=True)
>>> df
Animal Number_legs default_rank max_rank NA_bottom pct_rank
0 cat 4.0 2.5 3.0 2.5 0.625
@@ -10095,10 +9184,10 @@ def ranker(data):
@doc(_shared_docs["compare"], klass=_shared_doc_kwargs["klass"])
def compare(
self,
- other,
+ other: Self,
align_axis: Axis = 1,
- keep_shape: bool_t = False,
- keep_equal: bool_t = False,
+ keep_shape: bool = False,
+ keep_equal: bool = False,
result_names: Suffixes = ("self", "other"),
):
if type(self) is not type(other):
@@ -10107,7 +9196,8 @@ def compare(
f"can only compare '{cls_self}' (not '{cls_other}') with '{cls_self}'"
)
- mask = ~((self == other) | (self.isna() & other.isna()))
+ # error: Unsupported left operand type for & ("Self")
+ mask = ~((self == other) | (self.isna() & other.isna())) # type: ignore[operator]
mask.fillna(True, inplace=True)
if not keep_equal:
@@ -10182,12 +9272,8 @@ def align(
join: AlignJoin = "outer",
axis: Axis | None = None,
level: Level | None = None,
- copy: bool_t | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
fill_value: Hashable | None = None,
- method: FillnaOptions | None | lib.NoDefault = lib.no_default,
- limit: int | None | lib.NoDefault = lib.no_default,
- fill_axis: Axis | lib.NoDefault = lib.no_default,
- broadcast_axis: Axis | None | lib.NoDefault = lib.no_default,
) -> tuple[Self, NDFrameT]:
"""
Align two objects on their axes with the specified join method.
@@ -10197,6 +9283,7 @@ def align(
Parameters
----------
other : DataFrame or Series
+ The object to align with.
join : {{'outer', 'inner', 'left', 'right'}}, default 'outer'
Type of alignment to be performed.
@@ -10211,7 +9298,7 @@ def align(
level : int or level name, default None
Broadcast across a level, matching Index values on the
passed MultiIndex level.
- copy : bool, default True
+ copy : bool, default False
Always returns new objects. If copy=False and no reindexing is
required then original objects are returned.
@@ -10226,43 +9313,22 @@ def align(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+
+ .. deprecated:: 3.0.0
fill_value : scalar, default np.nan
Value to use for missing values. Defaults to NaN, but can be any
"compatible" value.
- method : {{'backfill', 'bfill', 'pad', 'ffill', None}}, default None
- Method to use for filling holes in reindexed Series:
-
- - pad / ffill: propagate last valid observation forward to next valid.
- - backfill / bfill: use NEXT valid observation to fill gap.
-
- .. deprecated:: 2.1
-
- limit : int, default None
- If method is specified, this is the maximum number of consecutive
- NaN values to forward/backward fill. In other words, if there is
- a gap with more than this number of consecutive NaNs, it will only
- be partially filled. If method is not specified, this is the
- maximum number of entries along the entire axis where NaNs will be
- filled. Must be greater than 0 if not None.
-
- .. deprecated:: 2.1
-
- fill_axis : {axes_single_arg}, default 0
- Filling axis, method and limit.
-
- .. deprecated:: 2.1
-
- broadcast_axis : {axes_single_arg}, default None
- Broadcast values along this axis, if aligning two objects of
- different dimensions.
-
- .. deprecated:: 2.1
Returns
-------
tuple of ({klass}, type of other)
Aligned objects.
+ See Also
+ --------
+ Series.align : Align two objects on their axes with specified join method.
+ DataFrame.align : Align two objects on their axes with specified join method.
+
Examples
--------
>>> df = pd.DataFrame(
@@ -10328,93 +9394,7 @@ def align(
3 60.0 70.0 80.0 90.0 NaN
4 600.0 700.0 800.0 900.0 NaN
"""
- if (
- method is not lib.no_default
- or limit is not lib.no_default
- or fill_axis is not lib.no_default
- ):
- # GH#51856
- warnings.warn(
- "The 'method', 'limit', and 'fill_axis' keywords in "
- f"{type(self).__name__}.align are deprecated and will be removed "
- "in a future version. Call fillna directly on the returned objects "
- "instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- if fill_axis is lib.no_default:
- fill_axis = 0
- if method is lib.no_default:
- method = None
- if limit is lib.no_default:
- limit = None
-
- if method is not None:
- method = clean_fill_method(method)
-
- if broadcast_axis is not lib.no_default:
- # GH#51856
- # TODO(3.0): enforcing this deprecation will close GH#13194
- msg = (
- f"The 'broadcast_axis' keyword in {type(self).__name__}.align is "
- "deprecated and will be removed in a future version."
- )
- if broadcast_axis is not None:
- if self.ndim == 1 and other.ndim == 2:
- msg += (
- " Use left = DataFrame({col: left for col in right.columns}, "
- "index=right.index) before calling `left.align(right)` instead."
- )
- elif self.ndim == 2 and other.ndim == 1:
- msg += (
- " Use right = DataFrame({col: right for col in left.columns}, "
- "index=left.index) before calling `left.align(right)` instead"
- )
- warnings.warn(msg, FutureWarning, stacklevel=find_stack_level())
- else:
- broadcast_axis = None
-
- if broadcast_axis == 1 and self.ndim != other.ndim:
- if isinstance(self, ABCSeries):
- # this means other is a DataFrame, and we need to broadcast
- # self
- cons = self._constructor_expanddim
- df = cons(
- {c: self for c in other.columns}, **other._construct_axes_dict()
- )
- # error: Incompatible return value type (got "Tuple[DataFrame,
- # DataFrame]", expected "Tuple[Self, NDFrameT]")
- return df._align_frame( # type: ignore[return-value]
- other, # type: ignore[arg-type]
- join=join,
- axis=axis,
- level=level,
- copy=copy,
- fill_value=fill_value,
- method=method,
- limit=limit,
- fill_axis=fill_axis,
- )[:2]
- elif isinstance(other, ABCSeries):
- # this means self is a DataFrame, and we need to broadcast
- # other
- cons = other._constructor_expanddim
- df = cons(
- {c: other for c in self.columns}, **self._construct_axes_dict()
- )
- # error: Incompatible return value type (got "Tuple[NDFrameT,
- # DataFrame]", expected "Tuple[Self, NDFrameT]")
- return self._align_frame( # type: ignore[return-value]
- df,
- join=join,
- axis=axis,
- level=level,
- copy=copy,
- fill_value=fill_value,
- method=method,
- limit=limit,
- fill_axis=fill_axis,
- )[:2]
+ self._check_copy_deprecation(copy)
_right: DataFrame | Series
if axis is not None:
@@ -10425,11 +9405,7 @@ def align(
join=join,
axis=axis,
level=level,
- copy=copy,
fill_value=fill_value,
- method=method,
- limit=limit,
- fill_axis=fill_axis,
)
elif isinstance(other, ABCSeries):
@@ -10438,11 +9414,7 @@ def align(
join=join,
axis=axis,
level=level,
- copy=copy,
fill_value=fill_value,
- method=method,
- limit=limit,
- fill_axis=fill_axis,
)
else: # pragma: no cover
raise TypeError(f"unsupported type: {type(other)}")
@@ -10472,11 +9444,7 @@ def _align_frame(
join: AlignJoin = "outer",
axis: Axis | None = None,
level=None,
- copy: bool_t | None = None,
fill_value=None,
- method=None,
- limit: int | None = None,
- fill_axis: Axis = 0,
) -> tuple[Self, DataFrame, Index | None]:
# defaults
join_index, join_columns = None, None
@@ -10505,20 +9473,14 @@ def _align_frame(
reindexers = {0: [join_index, ilidx], 1: [join_columns, clidx]}
left = self._reindex_with_indexers(
- reindexers, copy=copy, fill_value=fill_value, allow_dups=True
+ reindexers, fill_value=fill_value, allow_dups=True
)
# other must be always DataFrame
right = other._reindex_with_indexers(
{0: [join_index, iridx], 1: [join_columns, cridx]},
- copy=copy,
fill_value=fill_value,
allow_dups=True,
)
-
- if method is not None:
- left = left._pad_or_backfill(method, axis=fill_axis, limit=limit)
- right = right._pad_or_backfill(method, axis=fill_axis, limit=limit)
-
return left, right, join_index
@final
@@ -10528,15 +9490,9 @@ def _align_series(
join: AlignJoin = "outer",
axis: Axis | None = None,
level=None,
- copy: bool_t | None = None,
fill_value=None,
- method=None,
- limit: int | None = None,
- fill_axis: Axis = 0,
) -> tuple[Self, Series, Index | None]:
is_series = isinstance(self, ABCSeries)
- if copy and using_copy_on_write():
- copy = False
if (not is_series and axis is None) or axis not in [None, 0, 1]:
raise ValueError("Must specify axis=0 or 1")
@@ -10555,14 +9511,14 @@ def _align_series(
)
if is_series:
- left = self._reindex_indexer(join_index, lidx, copy)
+ left = self._reindex_indexer(join_index, lidx)
elif lidx is None or join_index is None:
- left = self.copy(deep=copy)
+ left = self.copy(deep=False)
else:
- new_mgr = self._mgr.reindex_indexer(join_index, lidx, axis=1, copy=copy)
+ new_mgr = self._mgr.reindex_indexer(join_index, lidx, axis=1)
left = self._constructor_from_mgr(new_mgr, axes=new_mgr.axes)
- right = other._reindex_indexer(join_index, ridx, copy)
+ right = other._reindex_indexer(join_index, ridx)
else:
# one has > 1 ndim
@@ -10578,39 +9534,64 @@ def _align_series(
bm_axis = self._get_block_manager_axis(1)
fdata = fdata.reindex_indexer(join_index, lidx, axis=bm_axis)
- if copy and fdata is self._mgr:
- fdata = fdata.copy()
-
left = self._constructor_from_mgr(fdata, axes=fdata.axes)
if ridx is None:
- right = other.copy(deep=copy)
+ right = other.copy(deep=False)
else:
right = other.reindex(join_index, level=level)
# fill
- fill_na = notna(fill_value) or (method is not None)
+ fill_na = notna(fill_value)
if fill_na:
- fill_value, method = validate_fillna_kwargs(fill_value, method)
- if method is not None:
- left = left._pad_or_backfill(method, limit=limit, axis=fill_axis)
- right = right._pad_or_backfill(method, limit=limit)
- else:
- left = left.fillna(fill_value, limit=limit, axis=fill_axis)
- right = right.fillna(fill_value, limit=limit)
+ left = left.fillna(fill_value)
+ right = right.fillna(fill_value)
return left, right, join_index
+ @overload
+ def _where(
+ self,
+ cond,
+ other=...,
+ *,
+ inplace: Literal[False] = ...,
+ axis: Axis | None = ...,
+ level=...,
+ ) -> Self: ...
+
+ @overload
+ def _where(
+ self,
+ cond,
+ other=...,
+ *,
+ inplace: Literal[True],
+ axis: Axis | None = ...,
+ level=...,
+ ) -> None: ...
+
+ @overload
+ def _where(
+ self,
+ cond,
+ other=...,
+ *,
+ inplace: bool,
+ axis: Axis | None = ...,
+ level=...,
+ ) -> Self | None: ...
+
@final
def _where(
self,
cond,
other=lib.no_default,
- inplace: bool_t = False,
+ *,
+ inplace: bool = False,
axis: Axis | None = None,
level=None,
- warn: bool_t = True,
- ):
+ ) -> Self | None:
"""
Equivalent to public method `where`, except that `other` is not
applied as a function even if callable. Used in __setitem__.
@@ -10630,7 +9611,7 @@ def _where(
copy=False,
)
cond.columns = self.columns
- cond = cond.align(self, join="right", copy=False)[0]
+ cond = cond.align(self, join="right")[0]
else:
if not hasattr(cond, "shape"):
cond = np.asanyarray(cond)
@@ -10640,14 +9621,8 @@ def _where(
# make sure we are boolean
fill_value = bool(inplace)
- with warnings.catch_warnings():
- warnings.filterwarnings(
- "ignore",
- "Downcasting object dtype arrays",
- category=FutureWarning,
- )
- cond = cond.fillna(fill_value)
- cond = cond.infer_objects(copy=False)
+ cond = cond.fillna(fill_value)
+ cond = cond.infer_objects()
msg = "Boolean array expected for the condition, not {dtype}"
@@ -10655,11 +9630,11 @@ def _where(
if not isinstance(cond, ABCDataFrame):
# This is a single-dimensional object.
if not is_bool_dtype(cond):
- raise ValueError(msg.format(dtype=cond.dtype))
+ raise TypeError(msg.format(dtype=cond.dtype))
else:
for _dt in cond.dtypes:
if not is_bool_dtype(_dt):
- raise ValueError(msg.format(dtype=_dt))
+ raise TypeError(msg.format(dtype=_dt))
if cond._mgr.any_extension_types:
# GH51574: avoid object ndarray conversion later on
cond = cond._constructor(
@@ -10671,7 +9646,7 @@ def _where(
cond = cond.astype(bool)
cond = -cond if inplace else cond
- cond = cond.reindex(self._info_axis, axis=self._info_axis_number, copy=False)
+ cond = cond.reindex(self._info_axis, axis=self._info_axis_number)
# try to align with other
if isinstance(other, NDFrame):
@@ -10684,7 +9659,6 @@ def _where(
axis=axis,
level=level,
fill_value=None,
- copy=False,
)[1]
# if we are NOT aligned, raise as we cannot where index
@@ -10740,7 +9714,7 @@ def _where(
# we may have different type blocks come out of putmask, so
# reconstruct the block manager
- new_data = self._mgr.putmask(mask=cond, new=other, align=align, warn=warn)
+ new_data = self._mgr.putmask(mask=cond, new=other, align=align)
result = self._constructor_from_mgr(new_data, axes=new_data.axes)
return self._update_inplace(result)
@@ -10762,8 +9736,7 @@ def where(
inplace: Literal[False] = ...,
axis: Axis | None = ...,
level: Level = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def where(
@@ -10774,8 +9747,7 @@ def where(
inplace: Literal[True],
axis: Axis | None = ...,
level: Level = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def where(
@@ -10783,11 +9755,10 @@ def where(
cond,
other=...,
*,
- inplace: bool_t = ...,
+ inplace: bool = ...,
axis: Axis | None = ...,
level: Level = ...,
- ) -> Self | None:
- ...
+ ) -> Self | None: ...
@final
@doc(
@@ -10802,7 +9773,7 @@ def where(
cond,
other=np.nan,
*,
- inplace: bool_t = False,
+ inplace: bool = False,
axis: Axis | None = None,
level: Level | None = None,
) -> Self | None:
@@ -10836,25 +9807,28 @@ def where(
Returns
-------
- Same type as caller or None if ``inplace=True``.
+ Series or DataFrame unless ``inplace=True`` in which case
+ returns None.
See Also
--------
:func:`DataFrame.{name_other}` : Return an object of same shape as
- self.
+ caller.
+ :func:`Series.{name_other}` : Return an object of same shape as
+ caller.
Notes
-----
The {name} method is an application of the if-then idiom. For each
- element in the calling DataFrame, if ``cond`` is ``{cond}`` the
- element is used; otherwise the corresponding element from the DataFrame
+ element in the caller, if ``cond`` is ``{cond}`` the
+ element is used; otherwise the corresponding element from
``other`` is used. If the axis of ``other`` does not align with axis of
- ``cond`` {klass}, the misaligned index positions will be filled with
- {cond_rev}.
+ ``cond`` {klass}, the values of ``cond`` on misaligned index positions
+ will be filled with {cond_rev}.
- The signature for :func:`DataFrame.where` differs from
- :func:`numpy.where`. Roughly ``df1.where(m, df2)`` is equivalent to
- ``np.where(m, df1, df2)``.
+ The signature for :func:`Series.where` or
+ :func:`DataFrame.where` differs from :func:`numpy.where`.
+ Roughly ``df1.where(m, df2)`` is equivalent to ``np.where(m, df1, df2)``.
For further details and examples see the ``{name}`` documentation in
:ref:`indexing `.
@@ -10912,7 +9886,7 @@ def where(
4 10
dtype: int64
- >>> df = pd.DataFrame(np.arange(10).reshape(-1, 2), columns=['A', 'B'])
+ >>> df = pd.DataFrame(np.arange(10).reshape(-1, 2), columns=["A", "B"])
>>> df
A B
0 0 1
@@ -10945,32 +9919,16 @@ def where(
"""
inplace = validate_bool_kwarg(inplace, "inplace")
if inplace:
- if not PYPY and using_copy_on_write():
+ if not PYPY:
if sys.getrefcount(self) <= REF_COUNT:
warnings.warn(
_chained_assignment_method_msg,
ChainedAssignmentError,
stacklevel=2,
)
- elif (
- not PYPY
- and not using_copy_on_write()
- and self._is_view_after_cow_rules()
- ):
- ctr = sys.getrefcount(self)
- ref_count = REF_COUNT
- if isinstance(self, ABCSeries) and hasattr(self, "_cacher"):
- # see https://github.com/pandas-dev/pandas/pull/56060#discussion_r1399245221
- ref_count += 1
- if ctr <= ref_count:
- warnings.warn(
- _chained_assignment_warning_method_msg,
- FutureWarning,
- stacklevel=2,
- )
other = common.apply_if_callable(other, self)
- return self._where(cond, other, inplace, axis, level)
+ return self._where(cond, other, inplace=inplace, axis=axis, level=level)
@overload
def mask(
@@ -10981,8 +9939,7 @@ def mask(
inplace: Literal[False] = ...,
axis: Axis | None = ...,
level: Level = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def mask(
@@ -10993,8 +9950,7 @@ def mask(
inplace: Literal[True],
axis: Axis | None = ...,
level: Level = ...,
- ) -> None:
- ...
+ ) -> None: ...
@overload
def mask(
@@ -11002,11 +9958,10 @@ def mask(
cond,
other=...,
*,
- inplace: bool_t = ...,
+ inplace: bool = ...,
axis: Axis | None = ...,
level: Level = ...,
- ) -> Self | None:
- ...
+ ) -> Self | None: ...
@final
@doc(
@@ -11022,35 +9977,19 @@ def mask(
cond,
other=lib.no_default,
*,
- inplace: bool_t = False,
+ inplace: bool = False,
axis: Axis | None = None,
level: Level | None = None,
) -> Self | None:
inplace = validate_bool_kwarg(inplace, "inplace")
if inplace:
- if not PYPY and using_copy_on_write():
+ if not PYPY:
if sys.getrefcount(self) <= REF_COUNT:
warnings.warn(
_chained_assignment_method_msg,
ChainedAssignmentError,
stacklevel=2,
)
- elif (
- not PYPY
- and not using_copy_on_write()
- and self._is_view_after_cow_rules()
- ):
- ctr = sys.getrefcount(self)
- ref_count = REF_COUNT
- if isinstance(self, ABCSeries) and hasattr(self, "_cacher"):
- # see https://github.com/pandas-dev/pandas/pull/56060#discussion_r1399245221
- ref_count += 1
- if ctr <= ref_count:
- warnings.warn(
- _chained_assignment_warning_method_msg,
- FutureWarning,
- stacklevel=2,
- )
cond = common.apply_if_callable(cond, self)
other = common.apply_if_callable(other, self)
@@ -11128,10 +10067,11 @@ def shift(
Examples
--------
- >>> df = pd.DataFrame({{"Col1": [10, 20, 15, 30, 45],
- ... "Col2": [13, 23, 18, 33, 48],
- ... "Col3": [17, 27, 22, 37, 52]}},
- ... index=pd.date_range("2020-01-01", "2020-01-05"))
+ >>> df = pd.DataFrame(
+ ... [[10, 13, 17], [20, 23, 27], [15, 18, 22], [30, 33, 37], [45, 48, 52]],
+ ... columns=["Col1", "Col2", "Col3"],
+ ... index=pd.date_range("2020-01-01", "2020-01-05"),
+ ... )
>>> df
Col1 Col2 Col3
2020-01-01 10 13 17
@@ -11180,7 +10120,7 @@ def shift(
2020-01-07 30 33 37
2020-01-08 45 48 52
- >>> df['Col1'].shift(periods=[0, 1, 2])
+ >>> df["Col1"].shift(periods=[0, 1, 2])
Col1_0 Col1_1 Col1_2
2020-01-01 10 NaN NaN
2020-01-02 20 10.0 NaN
@@ -11192,17 +10132,12 @@ def shift(
if freq is not None and fill_value is not lib.no_default:
# GH#53832
- warnings.warn(
- "Passing a 'freq' together with a 'fill_value' silently ignores "
- "the fill_value and is deprecated. This will raise in a future "
- "version.",
- FutureWarning,
- stacklevel=find_stack_level(),
+ raise ValueError(
+ "Passing a 'freq' together with a 'fill_value' is not allowed."
)
- fill_value = lib.no_default
if periods == 0:
- return self.copy(deep=None)
+ return self.copy(deep=False)
if is_list_like(periods) and isinstance(self, ABCSeries):
return self.to_frame().shift(
@@ -11246,11 +10181,11 @@ def _shift_with_freq(self, periods: int, axis: int, freq) -> Self:
if freq != orig_freq:
assert orig_freq is not None # for mypy
raise ValueError(
- f"Given freq {freq_to_period_freqstr(freq.n, freq.name)} "
+ f"Given freq {PeriodDtype(freq)._freqstr} "
f"does not match PeriodIndex freq "
- f"{freq_to_period_freqstr(orig_freq.n, orig_freq.name)}"
+ f"{PeriodDtype(orig_freq)._freqstr}"
)
- new_ax = index.shift(periods)
+ new_ax: Index = index.shift(periods)
else:
new_ax = index.shift(periods, freq)
@@ -11263,7 +10198,7 @@ def truncate(
before=None,
after=None,
axis: Axis | None = None,
- copy: bool_t | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
) -> Self:
"""
Truncate a Series or DataFrame before and after some index value.
@@ -11280,7 +10215,7 @@ def truncate(
axis : {0 or 'index', 1 or 'columns'}, optional
Axis to truncate. Truncates the index (rows) by default.
For `Series` this parameter is unused and defaults to 0.
- copy : bool, default is True,
+ copy : bool, default is False,
Return a copy of the truncated section.
.. note::
@@ -11295,6 +10230,8 @@ def truncate(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+ .. deprecated:: 3.0.0
+
Returns
-------
type of caller
@@ -11313,10 +10250,14 @@ def truncate(
Examples
--------
- >>> df = pd.DataFrame({'A': ['a', 'b', 'c', 'd', 'e'],
- ... 'B': ['f', 'g', 'h', 'i', 'j'],
- ... 'C': ['k', 'l', 'm', 'n', 'o']},
- ... index=[1, 2, 3, 4, 5])
+ >>> df = pd.DataFrame(
+ ... {
+ ... "A": ["a", "b", "c", "d", "e"],
+ ... "B": ["f", "g", "h", "i", "j"],
+ ... "C": ["k", "l", "m", "n", "o"],
+ ... },
+ ... index=[1, 2, 3, 4, 5],
+ ... )
>>> df
A B C
1 a f k
@@ -11343,7 +10284,7 @@ def truncate(
For Series, only rows can be truncated.
- >>> df['A'].truncate(before=2, after=4)
+ >>> df["A"].truncate(before=2, after=4)
2 b
3 c
4 d
@@ -11352,8 +10293,8 @@ def truncate(
The index values in ``truncate`` can be datetimes or string
dates.
- >>> dates = pd.date_range('2016-01-01', '2016-02-01', freq='s')
- >>> df = pd.DataFrame(index=dates, data={'A': 1})
+ >>> dates = pd.date_range("2016-01-01", "2016-02-01", freq="s")
+ >>> df = pd.DataFrame(index=dates, data={"A": 1})
>>> df.tail()
A
2016-01-31 23:59:56 1
@@ -11362,8 +10303,9 @@ def truncate(
2016-01-31 23:59:59 1
2016-02-01 00:00:00 1
- >>> df.truncate(before=pd.Timestamp('2016-01-05'),
- ... after=pd.Timestamp('2016-01-10')).tail()
+ >>> df.truncate(
+ ... before=pd.Timestamp("2016-01-05"), after=pd.Timestamp("2016-01-10")
+ ... ).tail()
A
2016-01-09 23:59:56 1
2016-01-09 23:59:57 1
@@ -11375,7 +10317,7 @@ def truncate(
specify `before` and `after` as strings. They will be coerced to
Timestamps before truncation.
- >>> df.truncate('2016-01-05', '2016-01-10').tail()
+ >>> df.truncate("2016-01-05", "2016-01-10").tail()
A
2016-01-09 23:59:56 1
2016-01-09 23:59:57 1
@@ -11387,7 +10329,7 @@ def truncate(
component (midnight). This differs from partial string slicing, which
returns any partially matching dates.
- >>> df.loc['2016-01-05':'2016-01-10', :].tail()
+ >>> df.loc["2016-01-05":"2016-01-10", :].tail()
A
2016-01-10 23:59:55 1
2016-01-10 23:59:56 1
@@ -11395,6 +10337,8 @@ def truncate(
2016-01-10 23:59:58 1
2016-01-10 23:59:59 1
"""
+ self._check_copy_deprecation(copy)
+
if axis is None:
axis = 0
axis = self._get_axis_number(axis)
@@ -11426,14 +10370,18 @@ def truncate(
if isinstance(ax, MultiIndex):
setattr(result, self._get_axis_name(axis), ax.truncate(before, after))
- result = result.copy(deep=copy and not using_copy_on_write())
+ result = result.copy(deep=False)
return result
@final
@doc(klass=_shared_doc_kwargs["klass"])
def tz_convert(
- self, tz, axis: Axis = 0, level=None, copy: bool_t | None = None
+ self,
+ tz,
+ axis: Axis = 0,
+ level=None,
+ copy: bool | lib.NoDefault = lib.no_default,
) -> Self:
"""
Convert tz-aware axis to target time zone.
@@ -11448,7 +10396,7 @@ def tz_convert(
level : int, str, default None
If axis is a MultiIndex, convert a specific level. Otherwise
must be None.
- copy : bool, default True
+ copy : bool, default False
Also make a copy of the underlying data.
.. note::
@@ -11463,6 +10411,8 @@ def tz_convert(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
+ .. deprecated:: 3.0.0
+
Returns
-------
{klass}
@@ -11479,20 +10429,20 @@ def tz_convert(
>>> s = pd.Series(
... [1],
- ... index=pd.DatetimeIndex(['2018-09-15 01:30:00+02:00']),
+ ... index=pd.DatetimeIndex(["2018-09-15 01:30:00+02:00"]),
... )
- >>> s.tz_convert('Asia/Shanghai')
+ >>> s.tz_convert("Asia/Shanghai")
2018-09-15 07:30:00+08:00 1
dtype: int64
Pass None to convert to UTC and get a tz-naive index:
- >>> s = pd.Series([1],
- ... index=pd.DatetimeIndex(['2018-09-15 01:30:00+02:00']))
+ >>> s = pd.Series([1], index=pd.DatetimeIndex(["2018-09-15 01:30:00+02:00"]))
>>> s.tz_convert(None)
2018-09-14 23:30:00 1
dtype: int64
"""
+ self._check_copy_deprecation(copy)
axis = self._get_axis_number(axis)
ax = self._get_axis(axis)
@@ -11519,8 +10469,8 @@ def _tz_convert(ax, tz):
raise ValueError(f"The level {level} is not valid")
ax = _tz_convert(ax, tz)
- result = self.copy(deep=copy and not using_copy_on_write())
- result = result.set_axis(ax, axis=axis, copy=False)
+ result = self.copy(deep=False)
+ result = result.set_axis(ax, axis=axis)
return result.__finalize__(self, method="tz_convert")
@final
@@ -11530,7 +10480,7 @@ def tz_localize(
tz,
axis: Axis = 0,
level=None,
- copy: bool_t | None = None,
+ copy: bool | lib.NoDefault = lib.no_default,
ambiguous: TimeAmbiguous = "raise",
nonexistent: TimeNonexistent = "raise",
) -> Self:
@@ -11550,7 +10500,7 @@ def tz_localize(
level : int, str, default None
If axis ia a MultiIndex, localize a specific level. Otherwise
must be None.
- copy : bool, default True
+ copy : bool, default False
Also make a copy of the underlying data.
.. note::
@@ -11564,7 +10514,9 @@ def tz_localize(
You can already get the future behavior and improvements through
enabling copy on write ``pd.options.mode.copy_on_write = True``
- ambiguous : 'infer', bool-ndarray, 'NaT', default 'raise'
+
+ .. deprecated:: 3.0.0
+ ambiguous : 'infer', bool, bool-ndarray, 'NaT', default 'raise'
When clocks moved backward due to DST, ambiguous times may arise.
For example in Central European Time (UTC+01), when going from
03:00 DST to 02:00 non-DST, 02:30:00 local time occurs both at
@@ -11574,7 +10526,7 @@ def tz_localize(
- 'infer' will attempt to infer fall dst-transition hours based on
order
- - bool-ndarray where True signifies a DST time, False designates
+ - bool (or bool-ndarray) where True signifies a DST time, False designates
a non-DST time (note that this flag is only applicable for
ambiguous times)
- 'NaT' will return NaT where there are ambiguous times
@@ -11609,16 +10561,15 @@ def tz_localize(
>>> s = pd.Series(
... [1],
- ... index=pd.DatetimeIndex(['2018-09-15 01:30:00']),
+ ... index=pd.DatetimeIndex(["2018-09-15 01:30:00"]),
... )
- >>> s.tz_localize('CET')
+ >>> s.tz_localize("CET")
2018-09-15 01:30:00+02:00 1
dtype: int64
Pass None to convert to tz-naive index and preserve local time:
- >>> s = pd.Series([1],
- ... index=pd.DatetimeIndex(['2018-09-15 01:30:00+02:00']))
+ >>> s = pd.Series([1], index=pd.DatetimeIndex(["2018-09-15 01:30:00+02:00"]))
>>> s.tz_localize(None)
2018-09-15 01:30:00 1
dtype: int64
@@ -11626,15 +10577,21 @@ def tz_localize(
Be careful with DST changes. When there is sequential data, pandas
can infer the DST time:
- >>> s = pd.Series(range(7),
- ... index=pd.DatetimeIndex(['2018-10-28 01:30:00',
- ... '2018-10-28 02:00:00',
- ... '2018-10-28 02:30:00',
- ... '2018-10-28 02:00:00',
- ... '2018-10-28 02:30:00',
- ... '2018-10-28 03:00:00',
- ... '2018-10-28 03:30:00']))
- >>> s.tz_localize('CET', ambiguous='infer')
+ >>> s = pd.Series(
+ ... range(7),
+ ... index=pd.DatetimeIndex(
+ ... [
+ ... "2018-10-28 01:30:00",
+ ... "2018-10-28 02:00:00",
+ ... "2018-10-28 02:30:00",
+ ... "2018-10-28 02:00:00",
+ ... "2018-10-28 02:30:00",
+ ... "2018-10-28 03:00:00",
+ ... "2018-10-28 03:30:00",
+ ... ]
+ ... ),
+ ... )
+ >>> s.tz_localize("CET", ambiguous="infer")
2018-10-28 01:30:00+02:00 0
2018-10-28 02:00:00+02:00 1
2018-10-28 02:30:00+02:00 2
@@ -11647,11 +10604,17 @@ def tz_localize(
In some cases, inferring the DST is impossible. In such cases, you can
pass an ndarray to the ambiguous parameter to set the DST explicitly
- >>> s = pd.Series(range(3),
- ... index=pd.DatetimeIndex(['2018-10-28 01:20:00',
- ... '2018-10-28 02:36:00',
- ... '2018-10-28 03:46:00']))
- >>> s.tz_localize('CET', ambiguous=np.array([True, True, False]))
+ >>> s = pd.Series(
+ ... range(3),
+ ... index=pd.DatetimeIndex(
+ ... [
+ ... "2018-10-28 01:20:00",
+ ... "2018-10-28 02:36:00",
+ ... "2018-10-28 03:46:00",
+ ... ]
+ ... ),
+ ... )
+ >>> s.tz_localize("CET", ambiguous=np.array([True, True, False]))
2018-10-28 01:20:00+02:00 0
2018-10-28 02:36:00+02:00 1
2018-10-28 03:46:00+01:00 2
@@ -11661,22 +10624,24 @@ def tz_localize(
dates forward or backward with a timedelta object or `'shift_forward'`
or `'shift_backward'`.
- >>> s = pd.Series(range(2),
- ... index=pd.DatetimeIndex(['2015-03-29 02:30:00',
- ... '2015-03-29 03:30:00']))
- >>> s.tz_localize('Europe/Warsaw', nonexistent='shift_forward')
+ >>> s = pd.Series(
+ ... range(2),
+ ... index=pd.DatetimeIndex(["2015-03-29 02:30:00", "2015-03-29 03:30:00"]),
+ ... )
+ >>> s.tz_localize("Europe/Warsaw", nonexistent="shift_forward")
2015-03-29 03:00:00+02:00 0
2015-03-29 03:30:00+02:00 1
dtype: int64
- >>> s.tz_localize('Europe/Warsaw', nonexistent='shift_backward')
+ >>> s.tz_localize("Europe/Warsaw", nonexistent="shift_backward")
2015-03-29 01:59:59.999999999+01:00 0
2015-03-29 03:30:00+02:00 1
dtype: int64
- >>> s.tz_localize('Europe/Warsaw', nonexistent=pd.Timedelta('1h'))
+ >>> s.tz_localize("Europe/Warsaw", nonexistent=pd.Timedelta("1h"))
2015-03-29 03:30:00+02:00 0
2015-03-29 03:30:00+02:00 1
dtype: int64
"""
+ self._check_copy_deprecation(copy)
nonexistent_options = ("raise", "NaT", "shift_forward", "shift_backward")
if nonexistent not in nonexistent_options and not isinstance(
nonexistent, dt.timedelta
@@ -11713,8 +10678,8 @@ def _tz_localize(ax, tz, ambiguous, nonexistent):
raise ValueError(f"The level {level} is not valid")
ax = _tz_localize(ax, tz, ambiguous, nonexistent)
- result = self.copy(deep=copy and not using_copy_on_write())
- result = result.set_axis(ax, axis=axis, copy=False)
+ result = self.copy(deep=False)
+ result = result.set_axis(ax, axis=axis)
return result.__finalize__(self, method="tz_localize")
# ----------------------------------------------------------------------
@@ -11795,17 +10760,17 @@ def describe(
upper percentile is ``75``. The ``50`` percentile is the
same as the median.
- For object data (e.g. strings or timestamps), the result's index
+ For object data (e.g. strings), the result's index
will include ``count``, ``unique``, ``top``, and ``freq``. The ``top``
is the most common value. The ``freq`` is the most common value's
- frequency. Timestamps also include the ``first`` and ``last`` items.
+ frequency.
If multiple object values have the highest count, then the
``count`` and ``top`` results will be arbitrarily chosen from
among those with the highest count.
For mixed data types provided via a ``DataFrame``, the default is to
- return only an analysis of numeric columns. If the dataframe consists
+ return only an analysis of numeric columns. If the DataFrame consists
only of object and categorical data without any numeric columns, the
default is to return an analysis of both the object and categorical
columns. If ``include='all'`` is provided as an option, the result
@@ -11833,7 +10798,7 @@ def describe(
Describing a categorical ``Series``.
- >>> s = pd.Series(['a', 'a', 'b', 'c'])
+ >>> s = pd.Series(["a", "a", "b", "c"])
>>> s.describe()
count 4
unique 3
@@ -11843,11 +10808,13 @@ def describe(
Describing a timestamp ``Series``.
- >>> s = pd.Series([
- ... np.datetime64("2000-01-01"),
- ... np.datetime64("2010-01-01"),
- ... np.datetime64("2010-01-01")
- ... ])
+ >>> s = pd.Series(
+ ... [
+ ... np.datetime64("2000-01-01"),
+ ... np.datetime64("2010-01-01"),
+ ... np.datetime64("2010-01-01"),
+ ... ]
+ ... )
>>> s.describe()
count 3
mean 2006-09-01 08:00:00
@@ -11861,10 +10828,13 @@ def describe(
Describing a ``DataFrame``. By default only numeric fields
are returned.
- >>> df = pd.DataFrame({'categorical': pd.Categorical(['d', 'e', 'f']),
- ... 'numeric': [1, 2, 3],
- ... 'object': ['a', 'b', 'c']
- ... })
+ >>> df = pd.DataFrame(
+ ... {
+ ... "categorical": pd.Categorical(["d", "e", "f"]),
+ ... "numeric": [1, 2, 3],
+ ... "object": ["a", "b", "c"],
+ ... }
+ ... )
>>> df.describe()
numeric
count 3.0
@@ -11878,7 +10848,7 @@ def describe(
Describing all columns of a ``DataFrame`` regardless of data type.
- >>> df.describe(include='all') # doctest: +SKIP
+ >>> df.describe(include="all") # doctest: +SKIP
categorical numeric object
count 3 3.0 3
unique 3 NaN 3
@@ -11930,7 +10900,7 @@ def describe(
Including only categorical columns from a ``DataFrame`` description.
- >>> df.describe(include=['category'])
+ >>> df.describe(include=["category"])
categorical
count 3
unique 3
@@ -11973,8 +10943,7 @@ def describe(
def pct_change(
self,
periods: int = 1,
- fill_method: FillnaOptions | None | lib.NoDefault = lib.no_default,
- limit: int | None | lib.NoDefault = lib.no_default,
+ fill_method: None = None,
freq=None,
**kwargs,
) -> Self:
@@ -11996,17 +10965,12 @@ def pct_change(
----------
periods : int, default 1
Periods to shift for forming percent change.
- fill_method : {'backfill', 'bfill', 'pad', 'ffill', None}, default 'pad'
- How to handle NAs **before** computing percent changes.
+ fill_method : None
+ Must be None. This argument will be removed in a future version of pandas.
.. deprecated:: 2.1
All options of `fill_method` are deprecated except `fill_method=None`.
- limit : int, default None
- The number of consecutive NAs to fill before stopping.
-
- .. deprecated:: 2.1
-
freq : DateOffset, timedelta, or str, optional
Increment to use from time series API (e.g. 'ME' or BDay()).
**kwargs
@@ -12071,11 +11035,14 @@ def pct_change(
Percentage change in French franc, Deutsche Mark, and Italian lira from
1980-01-01 to 1980-03-01.
- >>> df = pd.DataFrame({
- ... 'FR': [4.0405, 4.0963, 4.3149],
- ... 'GR': [1.7246, 1.7482, 1.8519],
- ... 'IT': [804.74, 810.01, 860.13]},
- ... index=['1980-01-01', '1980-02-01', '1980-03-01'])
+ >>> df = pd.DataFrame(
+ ... {
+ ... "FR": [4.0405, 4.0963, 4.3149],
+ ... "GR": [1.7246, 1.7482, 1.8519],
+ ... "IT": [804.74, 810.01, 860.13],
+ ... },
+ ... index=["1980-01-01", "1980-02-01", "1980-03-01"],
+ ... )
>>> df
FR GR IT
1980-01-01 4.0405 1.7246 804.74
@@ -12091,67 +11058,37 @@ def pct_change(
Percentage of change in GOOG and APPL stock volume. Shows computing
the percentage change between columns.
- >>> df = pd.DataFrame({
- ... '2016': [1769950, 30586265],
- ... '2015': [1500923, 40912316],
- ... '2014': [1371819, 41403351]},
- ... index=['GOOG', 'APPL'])
+ >>> df = pd.DataFrame(
+ ... {
+ ... "2016": [1769950, 30586265],
+ ... "2015": [1500923, 40912316],
+ ... "2014": [1371819, 41403351],
+ ... },
+ ... index=["GOOG", "APPL"],
+ ... )
>>> df
2016 2015 2014
GOOG 1769950 1500923 1371819
APPL 30586265 40912316 41403351
- >>> df.pct_change(axis='columns', periods=-1)
+ >>> df.pct_change(axis="columns", periods=-1)
2016 2015 2014
GOOG 0.179241 0.094112 NaN
APPL -0.252395 -0.011860 NaN
"""
# GH#53491
- if fill_method not in (lib.no_default, None) or limit is not lib.no_default:
- warnings.warn(
- "The 'fill_method' keyword being not None and the 'limit' keyword in "
- f"{type(self).__name__}.pct_change are deprecated and will be removed "
- "in a future version. Either fill in any non-leading NA values prior "
- "to calling pct_change or specify 'fill_method=None' to not fill NA "
- "values.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- if fill_method is lib.no_default:
- if limit is lib.no_default:
- cols = self.items() if self.ndim == 2 else [(None, self)]
- for _, col in cols:
- mask = col.isna().values
- mask = mask[np.argmax(~mask) :]
- if mask.any():
- warnings.warn(
- "The default fill_method='pad' in "
- f"{type(self).__name__}.pct_change is deprecated and will "
- "be removed in a future version. Either fill in any "
- "non-leading NA values prior to calling pct_change or "
- "specify 'fill_method=None' to not fill NA values.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- break
- fill_method = "pad"
- if limit is lib.no_default:
- limit = None
+ if fill_method is not None:
+ raise ValueError(f"fill_method must be None; got {fill_method=}.")
axis = self._get_axis_number(kwargs.pop("axis", "index"))
- if fill_method is None:
- data = self
- else:
- data = self._pad_or_backfill(fill_method, axis=axis, limit=limit)
-
- shifted = data.shift(periods=periods, freq=freq, axis=axis, **kwargs)
+ shifted = self.shift(periods=periods, freq=freq, axis=axis, **kwargs)
# Unsupported left operand type for / ("Self")
- rs = data / shifted - 1 # type: ignore[operator]
+ rs = self / shifted - 1 # type: ignore[operator]
if freq is not None:
# Shift method is implemented differently when freq is not None
# We want to restore the original index
rs = rs.loc[~rs.index.duplicated()]
- rs = rs.reindex_like(data)
+ rs = rs.reindex_like(self)
return rs.__finalize__(self, method="pct_change")
@final
@@ -12160,10 +11097,10 @@ def _logical_func(
name: str,
func,
axis: Axis | None = 0,
- bool_only: bool_t = False,
- skipna: bool_t = True,
+ bool_only: bool = False,
+ skipna: bool = True,
**kwargs,
- ) -> Series | bool_t:
+ ) -> Series | bool:
nv.validate_logical_func((), kwargs, fname=name)
validate_bool_kwarg(skipna, "skipna", none_allowed=False)
@@ -12204,22 +11141,24 @@ def _logical_func(
def any(
self,
+ *,
axis: Axis | None = 0,
- bool_only: bool_t = False,
- skipna: bool_t = True,
+ bool_only: bool = False,
+ skipna: bool = True,
**kwargs,
- ) -> Series | bool_t:
+ ) -> Series | bool:
return self._logical_func(
"any", nanops.nanany, axis, bool_only, skipna, **kwargs
)
def all(
self,
+ *,
axis: Axis = 0,
- bool_only: bool_t = False,
- skipna: bool_t = True,
+ bool_only: bool = False,
+ skipna: bool = True,
**kwargs,
- ) -> Series | bool_t:
+ ) -> Series | bool:
return self._logical_func(
"all", nanops.nanall, axis, bool_only, skipna, **kwargs
)
@@ -12230,7 +11169,7 @@ def _accum_func(
name: str,
func,
axis: Axis | None = None,
- skipna: bool_t = True,
+ skipna: bool = True,
*args,
**kwargs,
):
@@ -12242,7 +11181,12 @@ def _accum_func(
if axis == 1:
return self.T._accum_func(
- name, func, axis=0, skipna=skipna, *args, **kwargs # noqa: B026
+ name,
+ func,
+ axis=0,
+ skipna=skipna,
+ *args, # noqa: B026
+ **kwargs,
).T
def block_accum_func(blk_values):
@@ -12263,20 +11207,20 @@ def block_accum_func(blk_values):
self, method=name
)
- def cummax(self, axis: Axis | None = None, skipna: bool_t = True, *args, **kwargs):
+ def cummax(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
return self._accum_func(
"cummax", np.maximum.accumulate, axis, skipna, *args, **kwargs
)
- def cummin(self, axis: Axis | None = None, skipna: bool_t = True, *args, **kwargs):
+ def cummin(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
return self._accum_func(
"cummin", np.minimum.accumulate, axis, skipna, *args, **kwargs
)
- def cumsum(self, axis: Axis | None = None, skipna: bool_t = True, *args, **kwargs):
+ def cumsum(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
return self._accum_func("cumsum", np.cumsum, axis, skipna, *args, **kwargs)
- def cumprod(self, axis: Axis | None = None, skipna: bool_t = True, *args, **kwargs):
+ def cumprod(self, axis: Axis = 0, skipna: bool = True, *args, **kwargs) -> Self:
return self._accum_func("cumprod", np.cumprod, axis, skipna, *args, **kwargs)
@final
@@ -12284,39 +11228,26 @@ def _stat_function_ddof(
self,
name: str,
func,
- axis: Axis | None | lib.NoDefault = lib.no_default,
- skipna: bool_t = True,
+ axis: Axis | None = 0,
+ skipna: bool = True,
ddof: int = 1,
- numeric_only: bool_t = False,
+ numeric_only: bool = False,
**kwargs,
) -> Series | float:
nv.validate_stat_ddof_func((), kwargs, fname=name)
validate_bool_kwarg(skipna, "skipna", none_allowed=False)
- if axis is None:
- if self.ndim > 1:
- warnings.warn(
- f"The behavior of {type(self).__name__}.{name} with axis=None "
- "is deprecated, in a future version this will reduce over both "
- "axes and return a scalar. To retain the old behavior, pass "
- "axis=0 (or do not pass axis)",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- axis = 0
- elif axis is lib.no_default:
- axis = 0
-
return self._reduce(
func, name, axis=axis, numeric_only=numeric_only, skipna=skipna, ddof=ddof
)
def sem(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
+ skipna: bool = True,
ddof: int = 1,
- numeric_only: bool_t = False,
+ numeric_only: bool = False,
**kwargs,
) -> Series | float:
return self._stat_function_ddof(
@@ -12325,10 +11256,11 @@ def sem(
def var(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
+ skipna: bool = True,
ddof: int = 1,
- numeric_only: bool_t = False,
+ numeric_only: bool = False,
**kwargs,
) -> Series | float:
return self._stat_function_ddof(
@@ -12337,10 +11269,11 @@ def var(
def std(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
+ skipna: bool = True,
ddof: int = 1,
- numeric_only: bool_t = False,
+ numeric_only: bool = False,
**kwargs,
) -> Series | float:
return self._stat_function_ddof(
@@ -12353,8 +11286,8 @@ def _stat_function(
name: str,
func,
axis: Axis | None = 0,
- skipna: bool_t = True,
- numeric_only: bool_t = False,
+ skipna: bool = True,
+ numeric_only: bool = False,
**kwargs,
):
assert name in ["median", "mean", "min", "max", "kurt", "skew"], name
@@ -12368,9 +11301,10 @@ def _stat_function(
def min(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
- numeric_only: bool_t = False,
+ skipna: bool = True,
+ numeric_only: bool = False,
**kwargs,
):
return self._stat_function(
@@ -12384,9 +11318,10 @@ def min(
def max(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
- numeric_only: bool_t = False,
+ skipna: bool = True,
+ numeric_only: bool = False,
**kwargs,
):
return self._stat_function(
@@ -12400,9 +11335,10 @@ def max(
def mean(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
- numeric_only: bool_t = False,
+ skipna: bool = True,
+ numeric_only: bool = False,
**kwargs,
) -> Series | float:
return self._stat_function(
@@ -12411,9 +11347,10 @@ def mean(
def median(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
- numeric_only: bool_t = False,
+ skipna: bool = True,
+ numeric_only: bool = False,
**kwargs,
) -> Series | float:
return self._stat_function(
@@ -12422,9 +11359,10 @@ def median(
def skew(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
- numeric_only: bool_t = False,
+ skipna: bool = True,
+ numeric_only: bool = False,
**kwargs,
) -> Series | float:
return self._stat_function(
@@ -12433,9 +11371,10 @@ def skew(
def kurt(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
- numeric_only: bool_t = False,
+ skipna: bool = True,
+ numeric_only: bool = False,
**kwargs,
) -> Series | float:
return self._stat_function(
@@ -12449,9 +11388,9 @@ def _min_count_stat_function(
self,
name: str,
func,
- axis: Axis | None | lib.NoDefault = lib.no_default,
- skipna: bool_t = True,
- numeric_only: bool_t = False,
+ axis: Axis | None = 0,
+ skipna: bool = True,
+ numeric_only: bool = False,
min_count: int = 0,
**kwargs,
):
@@ -12460,20 +11399,6 @@ def _min_count_stat_function(
validate_bool_kwarg(skipna, "skipna", none_allowed=False)
- if axis is None:
- if self.ndim > 1:
- warnings.warn(
- f"The behavior of {type(self).__name__}.{name} with axis=None "
- "is deprecated, in a future version this will reduce over both "
- "axes and return a scalar. To retain the old behavior, pass "
- "axis=0 (or do not pass axis)",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- axis = 0
- elif axis is lib.no_default:
- axis = 0
-
return self._reduce(
func,
name=name,
@@ -12485,9 +11410,10 @@ def _min_count_stat_function(
def sum(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
- numeric_only: bool_t = False,
+ skipna: bool = True,
+ numeric_only: bool = False,
min_count: int = 0,
**kwargs,
):
@@ -12497,9 +11423,10 @@ def sum(
def prod(
self,
+ *,
axis: Axis | None = 0,
- skipna: bool_t = True,
- numeric_only: bool_t = False,
+ skipna: bool = True,
+ numeric_only: bool = False,
min_count: int = 0,
**kwargs,
):
@@ -12521,36 +11448,13 @@ def rolling(
self,
window: int | dt.timedelta | str | BaseOffset | BaseIndexer,
min_periods: int | None = None,
- center: bool_t = False,
+ center: bool = False,
win_type: str | None = None,
on: str | None = None,
- axis: Axis | lib.NoDefault = lib.no_default,
closed: IntervalClosedType | None = None,
step: int | None = None,
method: str = "single",
) -> Window | Rolling:
- if axis is not lib.no_default:
- axis = self._get_axis_number(axis)
- name = "rolling"
- if axis == 1:
- warnings.warn(
- f"Support for axis=1 in {type(self).__name__}.{name} is "
- "deprecated and will be removed in a future version. "
- f"Use obj.T.{name}(...) instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- warnings.warn(
- f"The 'axis' keyword in {type(self).__name__}.{name} is "
- "deprecated and will be removed in a future version. "
- "Call the method without the axis keyword instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- axis = 0
-
if win_type is not None:
return Window(
self,
@@ -12559,7 +11463,6 @@ def rolling(
center=center,
win_type=win_type,
on=on,
- axis=axis,
closed=closed,
step=step,
method=method,
@@ -12572,7 +11475,6 @@ def rolling(
center=center,
win_type=win_type,
on=on,
- axis=axis,
closed=closed,
step=step,
method=method,
@@ -12583,31 +11485,9 @@ def rolling(
def expanding(
self,
min_periods: int = 1,
- axis: Axis | lib.NoDefault = lib.no_default,
method: Literal["single", "table"] = "single",
) -> Expanding:
- if axis is not lib.no_default:
- axis = self._get_axis_number(axis)
- name = "expanding"
- if axis == 1:
- warnings.warn(
- f"Support for axis=1 in {type(self).__name__}.{name} is "
- "deprecated and will be removed in a future version. "
- f"Use obj.T.{name}(...) instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- warnings.warn(
- f"The 'axis' keyword in {type(self).__name__}.{name} is "
- "deprecated and will be removed in a future version. "
- "Call the method without the axis keyword instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- axis = 0
- return Expanding(self, min_periods=min_periods, axis=axis, method=method)
+ return Expanding(self, min_periods=min_periods, method=method)
@final
@doc(ExponentialMovingWindow)
@@ -12618,34 +11498,11 @@ def ewm(
halflife: float | TimedeltaConvertibleTypes | None = None,
alpha: float | None = None,
min_periods: int | None = 0,
- adjust: bool_t = True,
- ignore_na: bool_t = False,
- axis: Axis | lib.NoDefault = lib.no_default,
+ adjust: bool = True,
+ ignore_na: bool = False,
times: np.ndarray | DataFrame | Series | None = None,
method: Literal["single", "table"] = "single",
) -> ExponentialMovingWindow:
- if axis is not lib.no_default:
- axis = self._get_axis_number(axis)
- name = "ewm"
- if axis == 1:
- warnings.warn(
- f"Support for axis=1 in {type(self).__name__}.{name} is "
- "deprecated and will be removed in a future version. "
- f"Use obj.T.{name}(...) instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- warnings.warn(
- f"The 'axis' keyword in {type(self).__name__}.{name} is "
- "deprecated and will be removed in a future version. "
- "Call the method without the axis keyword instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- axis = 0
-
return ExponentialMovingWindow(
self,
com=com,
@@ -12655,7 +11512,6 @@ def ewm(
min_periods=min_periods,
adjust=adjust,
ignore_na=ignore_na,
- axis=axis,
times=times,
method=method,
)
@@ -12668,37 +11524,11 @@ def _inplace_method(self, other, op) -> Self:
"""
Wrap arithmetic method to operate inplace.
"""
- warn = True
- if not PYPY and warn_copy_on_write():
- if sys.getrefcount(self) <= REF_COUNT + 2:
- # we are probably in an inplace setitem context (e.g. df['a'] += 1)
- warn = False
-
result = op(self, other)
- if (
- self.ndim == 1
- and result._indexed_same(self)
- and result.dtype == self.dtype
- and not using_copy_on_write()
- and not (warn_copy_on_write() and not warn)
- ):
- # GH#36498 this inplace op can _actually_ be inplace.
- # Item "ArrayManager" of "Union[ArrayManager, SingleArrayManager,
- # BlockManager, SingleBlockManager]" has no attribute "setitem_inplace"
- self._mgr.setitem_inplace( # type: ignore[union-attr]
- slice(None), result._values, warn=warn
- )
- return self
-
- # Delete cacher
- self._reset_cacher()
-
# this makes sure that we are aligned like the input
- # we are updating inplace so we want to ignore is_copy
- self._update_inplace(
- result.reindex_like(self, copy=False), verify_is_copy=False
- )
+ # we are updating inplace
+ self._update_inplace(result.reindex_like(self))
return self
@final
@@ -12720,14 +11550,16 @@ def __imul__(self, other) -> Self:
def __itruediv__(self, other) -> Self:
# error: Unsupported left operand type for / ("Type[NDFrame]")
return self._inplace_method(
- other, type(self).__truediv__ # type: ignore[operator]
+ other,
+ type(self).__truediv__, # type: ignore[operator]
)
@final
def __ifloordiv__(self, other) -> Self:
# error: Unsupported left operand type for // ("Type[NDFrame]")
return self._inplace_method(
- other, type(self).__floordiv__ # type: ignore[operator]
+ other,
+ type(self).__floordiv__, # type: ignore[operator]
)
@final
@@ -12758,7 +11590,7 @@ def __ixor__(self, other) -> Self:
# Misc methods
@final
- def _find_valid_index(self, *, how: str) -> Hashable | None:
+ def _find_valid_index(self, *, how: str) -> Hashable:
"""
Retrieves the index of the first valid value.
@@ -12779,13 +11611,17 @@ def _find_valid_index(self, *, how: str) -> Hashable | None:
@final
@doc(position="first", klass=_shared_doc_kwargs["klass"])
- def first_valid_index(self) -> Hashable | None:
+ def first_valid_index(self) -> Hashable:
"""
- Return index for {position} non-NA value or None, if no non-NA value is found.
+ Return index for {position} non-missing value or None, if no value is found.
+
+ See the :ref:`User Guide ` for more information
+ on which values are considered missing.
Returns
-------
type of index
+ Index of {position} non-missing value.
Examples
--------
@@ -12815,7 +11651,7 @@ def first_valid_index(self) -> Hashable | None:
For DataFrame:
- >>> df = pd.DataFrame({{'A': [None, None, 2], 'B': [None, 3, 4]}})
+ >>> df = pd.DataFrame({{"A": [None, None, 2], "B": [None, 3, 4]}})
>>> df
A B
0 NaN NaN
@@ -12826,7 +11662,7 @@ def first_valid_index(self) -> Hashable | None:
>>> df.last_valid_index()
2
- >>> df = pd.DataFrame({{'A': [None, None, None], 'B': [None, None, None]}})
+ >>> df = pd.DataFrame({{"A": [None, None, None], "B": [None, None, None]}})
>>> df
A B
0 None None
@@ -12855,7 +11691,7 @@ def first_valid_index(self) -> Hashable | None:
@final
@doc(first_valid_index, position="last", klass=_shared_doc_kwargs["klass"])
- def last_valid_index(self) -> Hashable | None:
+ def last_valid_index(self) -> Hashable:
return self._find_valid_index(how="last")
@@ -13077,7 +11913,7 @@ def last_valid_index(self) -> Hashable | None:
**DataFrames**
-Create a dataframe from a dictionary.
+Create a DataFrame from a dictionary.
>>> df = pd.DataFrame({'col1': [True, True], 'col2': [True, False]})
>>> df
@@ -13495,9 +12331,7 @@ def last_valid_index(self) -> Hashable | None:
Series([], dtype: bool)
"""
-_shared_docs[
- "stat_func_example"
-] = """
+_shared_docs["stat_func_example"] = """
Examples
--------
diff --git a/pandas/core/groupby/base.py b/pandas/core/groupby/base.py
index a443597347283..bad9749b5ecee 100644
--- a/pandas/core/groupby/base.py
+++ b/pandas/core/groupby/base.py
@@ -1,6 +1,7 @@
"""
Provide basic components for groupby.
"""
+
from __future__ import annotations
import dataclasses
@@ -70,7 +71,6 @@ class OutputKey:
"cumsum",
"diff",
"ffill",
- "fillna",
"ngroup",
"pct_change",
"rank",
@@ -91,7 +91,6 @@ class OutputKey:
"corr",
"cov",
"describe",
- "dtypes",
"expanding",
"ewm",
"filter",
diff --git a/pandas/core/groupby/categorical.py b/pandas/core/groupby/categorical.py
index 6ab98cf4fe55e..49130d91a0126 100644
--- a/pandas/core/groupby/categorical.py
+++ b/pandas/core/groupby/categorical.py
@@ -10,9 +10,7 @@
)
-def recode_for_groupby(
- c: Categorical, sort: bool, observed: bool
-) -> tuple[Categorical, Categorical | None]:
+def recode_for_groupby(c: Categorical, sort: bool, observed: bool) -> Categorical:
"""
Code the categories to ensure we can groupby for categoricals.
@@ -42,15 +40,13 @@ def recode_for_groupby(
appearance in codes (unless ordered=True, in which case the
original order is preserved), followed by any unrepresented
categories in the original order.
- Categorical or None
- If we are observed, return the original categorical, otherwise None
"""
# we only care about observed values
if observed:
# In cases with c.ordered, this is equivalent to
# return c.remove_unused_categories(), c
- unique_codes = unique1d(c.codes)
+ unique_codes = unique1d(c.codes) # type: ignore[no-untyped-call]
take_codes = unique_codes[unique_codes != -1]
if sort:
@@ -63,18 +59,18 @@ def recode_for_groupby(
# return a new categorical that maps our new codes
# and categories
dtype = CategoricalDtype(categories, ordered=c.ordered)
- return Categorical._simple_new(codes, dtype=dtype), c
+ return Categorical._simple_new(codes, dtype=dtype)
# Already sorted according to c.categories; all is fine
if sort:
- return c, None
+ return c
# sort=False should order groups in as-encountered order (GH-8868)
# xref GH:46909: Re-ordering codes faster than using (set|add|reorder)_categories
all_codes = np.arange(c.categories.nunique())
# GH 38140: exclude nan from indexer for categories
- unique_notnan_codes = unique1d(c.codes[c.codes != -1])
+ unique_notnan_codes = unique1d(c.codes[c.codes != -1]) # type: ignore[no-untyped-call]
if sort:
unique_notnan_codes = np.sort(unique_notnan_codes)
if len(all_codes) > len(unique_notnan_codes):
@@ -84,4 +80,4 @@ def recode_for_groupby(
else:
take_codes = unique_notnan_codes
- return Categorical(c, c.unique().categories.take(take_codes)), None
+ return Categorical(c, c.unique().categories.take(take_codes))
diff --git a/pandas/core/groupby/generic.py b/pandas/core/groupby/generic.py
index f2e314046fb74..361e9e87fadb8 100644
--- a/pandas/core/groupby/generic.py
+++ b/pandas/core/groupby/generic.py
@@ -5,6 +5,7 @@
These are user facing as the result of the ``df.groupby(...)`` operations,
which here returns a DataFrameGroupBy object.
"""
+
from __future__ import annotations
from collections import abc
@@ -20,14 +21,10 @@
Union,
cast,
)
-import warnings
import numpy as np
-from pandas._libs import (
- Interval,
- lib,
-)
+from pandas._libs import Interval
from pandas._libs.hashtable import duplicated
from pandas.errors import SpecificationError
from pandas.util._decorators import (
@@ -35,7 +32,6 @@
Substitution,
doc,
)
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.common import (
ensure_int64,
@@ -62,20 +58,15 @@
maybe_mangle_lambdas,
reconstruct_func,
validate_func_kwargs,
- warn_alias_replacement,
)
import pandas.core.common as com
from pandas.core.frame import DataFrame
-from pandas.core.groupby import (
- base,
- ops,
-)
+from pandas.core.groupby import base
from pandas.core.groupby.groupby import (
GroupBy,
GroupByPlot,
_agg_template_frame,
_agg_template_series,
- _apply_docs,
_transform_template,
)
from pandas.core.indexes.api import (
@@ -93,20 +84,16 @@
if TYPE_CHECKING:
from collections.abc import (
Hashable,
- Mapping,
Sequence,
)
from pandas._typing import (
ArrayLike,
- Axis,
- AxisInt,
+ BlockManager,
CorrelationMethod,
- FillnaOptions,
IndexLabel,
Manager,
- Manager2D,
- SingleManager,
+ SingleBlockManager,
TakeIndexer,
)
@@ -159,7 +146,7 @@ def _wrap_agged_manager(self, mgr: Manager) -> Series:
def _get_data_to_aggregate(
self, *, numeric_only: bool = False, name: str | None = None
- ) -> SingleManager:
+ ) -> SingleBlockManager:
ser = self._obj_with_exclusions
single = ser._mgr
if numeric_only and not is_numeric_dtype(ser.dtype):
@@ -221,12 +208,113 @@ def _get_data_to_aggregate(
"""
)
- @Appender(
- _apply_docs["template"].format(
- input="series", examples=_apply_docs["series_examples"]
- )
- )
def apply(self, func, *args, **kwargs) -> Series:
+ """
+ Apply function ``func`` group-wise and combine the results together.
+
+ The function passed to ``apply`` must take a series as its first
+ argument and return a DataFrame, Series or scalar. ``apply`` will
+ then take care of combining the results back together into a single
+ dataframe or series. ``apply`` is therefore a highly flexible
+ grouping method.
+
+ While ``apply`` is a very flexible method, its downside is that
+ using it can be quite a bit slower than using more specific methods
+ like ``agg`` or ``transform``. Pandas offers a wide range of method that will
+ be much faster than using ``apply`` for their specific purposes, so try to
+ use them before reaching for ``apply``.
+
+ Parameters
+ ----------
+ func : callable
+ A callable that takes a series as its first argument, and
+ returns a dataframe, a series or a scalar. In addition the
+ callable may take positional and keyword arguments.
+
+ *args : tuple
+ Optional positional arguments to pass to ``func``.
+
+ **kwargs : dict
+ Optional keyword arguments to pass to ``func``.
+
+ Returns
+ -------
+ Series or DataFrame
+
+ See Also
+ --------
+ pipe : Apply function to the full GroupBy object instead of to each
+ group.
+ aggregate : Apply aggregate function to the GroupBy object.
+ transform : Apply function column-by-column to the GroupBy object.
+ Series.apply : Apply a function to a Series.
+ DataFrame.apply : Apply a function to each row or column of a DataFrame.
+
+ Notes
+ -----
+
+ .. versionchanged:: 1.3.0
+
+ The resulting dtype will reflect the return value of the passed ``func``,
+ see the examples below.
+
+ Functions that mutate the passed object can produce unexpected
+ behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
+ for more details.
+
+ Examples
+ --------
+ >>> s = pd.Series([0, 1, 2], index="a a b".split())
+ >>> g1 = s.groupby(s.index, group_keys=False)
+ >>> g2 = s.groupby(s.index, group_keys=True)
+
+ From ``s`` above we can see that ``g`` has two groups, ``a`` and ``b``.
+ Notice that ``g1`` have ``g2`` have two groups, ``a`` and ``b``, and only
+ differ in their ``group_keys`` argument. Calling `apply` in various ways,
+ we can get different grouping results:
+
+ Example 1: The function passed to `apply` takes a Series as
+ its argument and returns a Series. `apply` combines the result for
+ each group together into a new Series.
+
+ .. versionchanged:: 1.3.0
+
+ The resulting dtype will reflect the return value of the passed ``func``.
+
+ >>> g1.apply(lambda x: x * 2 if x.name == "a" else x / 2)
+ a 0.0
+ a 2.0
+ b 1.0
+ dtype: float64
+
+ In the above, the groups are not part of the index. We can have them included
+ by using ``g2`` where ``group_keys=True``:
+
+ >>> g2.apply(lambda x: x * 2 if x.name == "a" else x / 2)
+ a a 0.0
+ a 2.0
+ b b 1.0
+ dtype: float64
+
+ Example 2: The function passed to `apply` takes a Series as
+ its argument and returns a scalar. `apply` combines the result for
+ each group together into a Series, including setting the index as
+ appropriate:
+
+ >>> g1.apply(lambda x: x.max() - x.min())
+ a 1
+ b 0
+ dtype: int64
+
+ The ``group_keys`` argument has no effect here because the result is not
+ like-indexed (i.e. :ref:`a transform `) when compared
+ to the input.
+
+ >>> g2.apply(lambda x: x.max() - x.min())
+ a 1
+ b 0
+ dtype: int64
+ """
return super().apply(func, *args, **kwargs)
@doc(_agg_template_series, examples=_agg_examples_doc, klass="Series")
@@ -264,11 +352,6 @@ def aggregate(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs)
return ret
else:
- cyfunc = com.get_cython_func(func)
- if cyfunc and not args and not kwargs:
- warn_alias_replacement(self, func, cyfunc)
- return getattr(self, cyfunc)()
-
if maybe_use_numba(engine):
return self._aggregate_with_numba(
func, *args, engine_kwargs=engine_kwargs, **kwargs
@@ -286,41 +369,11 @@ def aggregate(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs)
index=self._grouper.result_index,
dtype=obj.dtype,
)
-
- if self._grouper.nkeys > 1:
- return self._python_agg_general(func, *args, **kwargs)
-
- try:
- return self._python_agg_general(func, *args, **kwargs)
- except KeyError:
- # KeyError raised in test_groupby.test_basic is bc the func does
- # a dictionary lookup on group.name, but group name is not
- # pinned in _python_agg_general, only in _aggregate_named
- result = self._aggregate_named(func, *args, **kwargs)
-
- warnings.warn(
- "Pinning the groupby key to each group in "
- f"{type(self).__name__}.agg is deprecated, and cases that "
- "relied on it will raise in a future version. "
- "If your operation requires utilizing the groupby keys, "
- "iterate over the groupby object instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- # result is a dict whose keys are the elements of result_index
- result = Series(result, index=self._grouper.result_index)
- result = self._wrap_aggregated_output(result)
- return result
+ return self._python_agg_general(func, *args, **kwargs)
agg = aggregate
def _python_agg_general(self, func, *args, **kwargs):
- orig_func = func
- func = com.is_builtin_func(func)
- if orig_func != func:
- alias = com._builtin_table_alias[func]
- warn_alias_replacement(self, orig_func, alias)
f = lambda x: func(x, *args, **kwargs)
obj = self._obj_with_exclusions
@@ -330,23 +383,9 @@ def _python_agg_general(self, func, *args, **kwargs):
def _aggregate_multiple_funcs(self, arg, *args, **kwargs) -> DataFrame:
if isinstance(arg, dict):
- if self.as_index:
- # GH 15931
- raise SpecificationError("nested renamer is not supported")
- else:
- # GH#50684 - This accidentally worked in 1.x
- msg = (
- "Passing a dictionary to SeriesGroupBy.agg is deprecated "
- "and will raise in a future version of pandas. Pass a list "
- "of aggregations instead."
- )
- warnings.warn(
- message=msg,
- category=FutureWarning,
- stacklevel=find_stack_level(),
- )
- arg = list(arg.items())
- elif any(isinstance(x, (tuple, list)) for x in arg):
+ raise SpecificationError("nested renamer is not supported")
+
+ if any(isinstance(x, (tuple, list)) for x in arg):
arg = [(x, x) if not isinstance(x, (tuple, list)) else x for x in arg]
else:
# list of functions / function names
@@ -418,10 +457,9 @@ def _wrap_applied_output(
# GH #823 #24880
index = self._grouper.result_index
res_df = self.obj._constructor_expanddim(values, index=index)
- res_df = self._reindex_output(res_df)
# if self.observed is False,
# keep all-NaN rows created while re-indexing
- res_ser = res_df.stack(future_stack=True)
+ res_ser = res_df.stack()
res_ser.name = self.obj.name
return res_ser
elif isinstance(values[0], (Series, DataFrame)):
@@ -444,29 +482,7 @@ def _wrap_applied_output(
if not self.as_index:
result = self._insert_inaxis_grouper(result)
result.index = default_index(len(result))
- return self._reindex_output(result)
-
- def _aggregate_named(self, func, *args, **kwargs):
- # Note: this is very similar to _aggregate_series_pure_python,
- # but that does not pin group.name
- result = {}
- initialized = False
-
- for name, group in self._grouper.get_iterator(
- self._obj_with_exclusions, axis=self.axis
- ):
- # needed for pandas/tests/groupby/test_groupby.py::test_basic_aggregations
- object.__setattr__(group, "name", name)
-
- output = func(group, *args, **kwargs)
- output = ops.extract_result(output)
- if not initialized:
- # We only do this validation on the first iteration
- ops.check_result_array(output, group.dtype)
- initialized = True
- result[name] = output
-
- return result
+ return result
__examples_series_doc = dedent(
"""
@@ -518,16 +534,12 @@ def transform(self, func, *args, engine=None, engine_kwargs=None, **kwargs):
func, *args, engine=engine, engine_kwargs=engine_kwargs, **kwargs
)
- def _cython_transform(
- self, how: str, numeric_only: bool = False, axis: AxisInt = 0, **kwargs
- ):
- assert axis == 0 # handled by caller
-
+ def _cython_transform(self, how: str, numeric_only: bool = False, **kwargs):
obj = self._obj_with_exclusions
try:
result = self._grouper._cython_operation(
- "transform", obj._values, how, axis, **kwargs
+ "transform", obj._values, how, 0, **kwargs
)
except NotImplementedError as err:
# e.g. test_groupby_raises_string
@@ -550,7 +562,7 @@ def _transform_general(
results = []
for name, group in self._grouper.get_iterator(
- self._obj_with_exclusions, axis=self.axis
+ self._obj_with_exclusions,
):
# this setattr is needed for test_transform_lambda_with_datetimetz
object.__setattr__(group, "name", name)
@@ -562,7 +574,7 @@ def _transform_general(
if results:
from pandas.core.reshape.concat import concat
- concatenated = concat(results)
+ concatenated = concat(results, ignore_index=True)
result = self._set_result_index_ordered(concatenated)
else:
result = self.obj._constructor(dtype=np.float64)
@@ -597,12 +609,15 @@ def filter(self, func, dropna: bool = True, *args, **kwargs):
Examples
--------
- >>> df = pd.DataFrame({'A' : ['foo', 'bar', 'foo', 'bar',
- ... 'foo', 'bar'],
- ... 'B' : [1, 2, 3, 4, 5, 6],
- ... 'C' : [2.0, 5., 8., 1., 2., 9.]})
- >>> grouped = df.groupby('A')
- >>> df.groupby('A').B.filter(lambda x: x.mean() > 3.)
+ >>> df = pd.DataFrame(
+ ... {
+ ... "A": ["foo", "bar", "foo", "bar", "foo", "bar"],
+ ... "B": [1, 2, 3, 4, 5, 6],
+ ... "C": [2.0, 5.0, 8.0, 1.0, 2.0, 9.0],
+ ... }
+ ... )
+ >>> grouped = df.groupby("A")
+ >>> df.groupby("A").B.filter(lambda x: x.mean() > 3.0)
1 2
3 4
5 6
@@ -621,9 +636,7 @@ def true_and_notna(x) -> bool:
try:
indices = [
self._get_index(name)
- for name, group in self._grouper.get_iterator(
- self._obj_with_exclusions, axis=self.axis
- )
+ for name, group in self._grouper.get_iterator(self._obj_with_exclusions)
if true_and_notna(group)
]
except (ValueError, TypeError) as err:
@@ -636,16 +649,23 @@ def nunique(self, dropna: bool = True) -> Series | DataFrame:
"""
Return number of unique elements in the group.
+ Parameters
+ ----------
+ dropna : bool, default True
+ Don't include NaN in the counts.
+
Returns
-------
Series
Number of unique values within each group.
- Examples
+ See Also
--------
- For SeriesGroupby:
+ core.resample.Resampler.nunique : Method nunique for Resampler.
- >>> lst = ['a', 'a', 'b', 'b']
+ Examples
+ --------
+ >>> lst = ["a", "a", "b", "b"]
>>> ser = pd.Series([1, 2, 3, 3], index=lst)
>>> ser
a 1
@@ -657,23 +677,8 @@ def nunique(self, dropna: bool = True) -> Series | DataFrame:
a 2
b 1
dtype: int64
-
- For Resampler:
-
- >>> ser = pd.Series([1, 2, 3, 3], index=pd.DatetimeIndex(
- ... ['2023-01-01', '2023-01-15', '2023-02-01', '2023-02-15']))
- >>> ser
- 2023-01-01 1
- 2023-01-15 2
- 2023-02-01 3
- 2023-02-15 3
- dtype: int64
- >>> ser.resample('MS').nunique()
- 2023-01-01 2
- 2023-02-01 1
- Freq: MS, dtype: int64
"""
- ids, _, ngroups = self._grouper.group_info
+ ids, ngroups = self._grouper.group_info
val = self.obj._values
codes, uniques = algorithms.factorize(val, use_na_sentinel=dropna, sort=False)
@@ -706,7 +711,7 @@ def nunique(self, dropna: bool = True) -> Series | DataFrame:
if not self.as_index:
result = self._insert_inaxis_grouper(result)
result.index = default_index(len(result))
- return self._reindex_output(result, fill_value=0)
+ return result
@doc(Series.describe)
def describe(self, percentiles=None, include=None, exclude=None) -> Series:
@@ -722,6 +727,85 @@ def value_counts(
bins=None,
dropna: bool = True,
) -> Series | DataFrame:
+ """
+ Return a Series or DataFrame containing counts of unique rows.
+
+ .. versionadded:: 1.4.0
+
+ Parameters
+ ----------
+ normalize : bool, default False
+ Return proportions rather than frequencies.
+ sort : bool, default True
+ Sort by frequencies.
+ ascending : bool, default False
+ Sort in ascending order.
+ bins : int or list of ints, optional
+ Rather than count values, group them into half-open bins,
+ a convenience for pd.cut, only works with numeric data.
+ dropna : bool, default True
+ Don't include counts of rows that contain NA values.
+
+ Returns
+ -------
+ Series or DataFrame
+ Series if the groupby ``as_index`` is True, otherwise DataFrame.
+
+ See Also
+ --------
+ Series.value_counts: Equivalent method on Series.
+ DataFrame.value_counts: Equivalent method on DataFrame.
+ DataFrameGroupBy.value_counts: Equivalent method on DataFrameGroupBy.
+
+ Notes
+ -----
+ - If the groupby ``as_index`` is True then the returned Series will have a
+ MultiIndex with one level per input column.
+ - If the groupby ``as_index`` is False then the returned DataFrame will have an
+ additional column with the value_counts. The column is labelled 'count' or
+ 'proportion', depending on the ``normalize`` parameter.
+
+ By default, rows that contain any NA values are omitted from
+ the result.
+
+ By default, the result will be in descending order so that the
+ first element of each group is the most frequently-occurring row.
+
+ Examples
+ --------
+ >>> s = pd.Series(
+ ... [1, 1, 2, 3, 2, 3, 3, 1, 1, 3, 3, 3],
+ ... index=["A", "A", "A", "A", "A", "A", "B", "B", "B", "B", "B", "B"],
+ ... )
+ >>> s
+ A 1
+ A 1
+ A 2
+ A 3
+ A 2
+ A 3
+ B 3
+ B 1
+ B 1
+ B 3
+ B 3
+ B 3
+ dtype: int64
+ >>> g1 = s.groupby(s.index)
+ >>> g1.value_counts(bins=2)
+ A (0.997, 2.0] 4
+ (2.0, 3.0] 2
+ B (2.0, 3.0] 4
+ (0.997, 2.0] 2
+ Name: count, dtype: int64
+ >>> g1.value_counts(normalize=True)
+ A 1 0.333333
+ 2 0.333333
+ 3 0.333333
+ B 3 0.666667
+ 1 0.333333
+ Name: proportion, dtype: float64
+ """
name = "proportion" if normalize else "count"
if bins is None:
@@ -734,7 +818,7 @@ def value_counts(
from pandas.core.reshape.merge import get_join_indexers
from pandas.core.reshape.tile import cut
- ids, _, _ = self._grouper.group_info
+ ids = self._grouper.ids
val = self.obj._values
index_names = self._grouper.names + [self.obj.name]
@@ -804,9 +888,18 @@ def value_counts(
rep = partial(np.repeat, repeats=np.add.reduceat(inc, idx))
# multi-index components
- codes = self._grouper.reconstructed_codes
+ if isinstance(self._grouper.result_index, MultiIndex):
+ codes = list(self._grouper.result_index.codes)
+ else:
+ codes = [
+ algorithms.factorize(
+ self._grouper.result_index,
+ sort=self._grouper._sort,
+ use_na_sentinel=self._grouper.dropna,
+ )[0]
+ ]
codes = [rep(level_codes) for level_codes in codes] + [llab(lab, inc)]
- levels = [ping._group_index for ping in self._grouper.groupings] + [lev]
+ levels = self._grouper.levels + [lev]
if dropna:
mask = codes[-1] != -1
@@ -848,7 +941,10 @@ def value_counts(
# "List[ndarray[Any, Any]]"; expected "List[Union[Union[ExtensionArray,
# ndarray[Any, Any]], Index, Series]]
_, idx = get_join_indexers(
- left, right, sort=False, how="left" # type: ignore[arg-type]
+ left, # type: ignore[arg-type]
+ right,
+ sort=False,
+ how="left",
)
if idx is not None:
out = np.where(idx != -1, out[idx], 0)
@@ -875,108 +971,9 @@ def build_codes(lev_codes: np.ndarray) -> np.ndarray:
result = result.reset_index()
return result
- def fillna(
- self,
- value: object | ArrayLike | None = None,
- method: FillnaOptions | None = None,
- axis: Axis | None | lib.NoDefault = lib.no_default,
- inplace: bool = False,
- limit: int | None = None,
- downcast: dict | None | lib.NoDefault = lib.no_default,
- ) -> Series | None:
- """
- Fill NA/NaN values using the specified method within groups.
-
- .. deprecated:: 2.2.0
- This method is deprecated and will be removed in a future version.
- Use the :meth:`.SeriesGroupBy.ffill` or :meth:`.SeriesGroupBy.bfill`
- for forward or backward filling instead. If you want to fill with a
- single value, use :meth:`Series.fillna` instead.
-
- Parameters
- ----------
- value : scalar, dict, Series, or DataFrame
- Value to use to fill holes (e.g. 0), alternately a
- dict/Series/DataFrame of values specifying which value to use for
- each index (for a Series) or column (for a DataFrame). Values not
- in the dict/Series/DataFrame will not be filled. This value cannot
- be a list. Users wanting to use the ``value`` argument and not ``method``
- should prefer :meth:`.Series.fillna` as this
- will produce the same result and be more performant.
- method : {{'bfill', 'ffill', None}}, default None
- Method to use for filling holes. ``'ffill'`` will propagate
- the last valid observation forward within a group.
- ``'bfill'`` will use next valid observation to fill the gap.
- axis : {0 or 'index', 1 or 'columns'}
- Unused, only for compatibility with :meth:`DataFrameGroupBy.fillna`.
- inplace : bool, default False
- Broken. Do not set to True.
- limit : int, default None
- If method is specified, this is the maximum number of consecutive
- NaN values to forward/backward fill within a group. In other words,
- if there is a gap with more than this number of consecutive NaNs,
- it will only be partially filled. If method is not specified, this is the
- maximum number of entries along the entire axis where NaNs will be
- filled. Must be greater than 0 if not None.
- downcast : dict, default is None
- A dict of item->dtype of what to downcast if possible,
- or the string 'infer' which will try to downcast to an appropriate
- equal type (e.g. float64 to int64 if possible).
-
- Returns
- -------
- Series
- Object with missing values filled within groups.
-
- See Also
- --------
- ffill : Forward fill values within a group.
- bfill : Backward fill values within a group.
-
- Examples
- --------
- For SeriesGroupBy:
-
- >>> lst = ['cat', 'cat', 'cat', 'mouse', 'mouse']
- >>> ser = pd.Series([1, None, None, 2, None], index=lst)
- >>> ser
- cat 1.0
- cat NaN
- cat NaN
- mouse 2.0
- mouse NaN
- dtype: float64
- >>> ser.groupby(level=0).fillna(0, limit=1)
- cat 1.0
- cat 0.0
- cat NaN
- mouse 2.0
- mouse 0.0
- dtype: float64
- """
- warnings.warn(
- f"{type(self).__name__}.fillna is deprecated and "
- "will be removed in a future version. Use obj.ffill() or obj.bfill() "
- "for forward or backward filling instead. If you want to fill with a "
- f"single value, use {type(self.obj).__name__}.fillna instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- result = self._op_via_apply(
- "fillna",
- value=value,
- method=method,
- axis=axis,
- inplace=inplace,
- limit=limit,
- downcast=downcast,
- )
- return result
-
def take(
self,
indices: TakeIndexer,
- axis: Axis | lib.NoDefault = lib.no_default,
**kwargs,
) -> Series:
"""
@@ -994,14 +991,6 @@ def take(
----------
indices : array-like
An array of ints indicating which positions to take in each group.
- axis : {0 or 'index', 1 or 'columns', None}, default 0
- The axis on which to select elements. ``0`` means that we are
- selecting rows, ``1`` means that we are selecting columns.
- For `SeriesGroupBy` this parameter is unused and defaults to 0.
-
- .. deprecated:: 2.1.0
- For axis=1, operate on the underlying object instead. Otherwise
- the axis keyword is not necessary.
**kwargs
For compatibility with :meth:`numpy.take`. Has no effect on the
@@ -1022,13 +1011,17 @@ def take(
Examples
--------
- >>> df = pd.DataFrame([('falcon', 'bird', 389.0),
- ... ('parrot', 'bird', 24.0),
- ... ('lion', 'mammal', 80.5),
- ... ('monkey', 'mammal', np.nan),
- ... ('rabbit', 'mammal', 15.0)],
- ... columns=['name', 'class', 'max_speed'],
- ... index=[4, 3, 2, 1, 0])
+ >>> df = pd.DataFrame(
+ ... [
+ ... ("falcon", "bird", 389.0),
+ ... ("parrot", "bird", 24.0),
+ ... ("lion", "mammal", 80.5),
+ ... ("monkey", "mammal", np.nan),
+ ... ("rabbit", "mammal", 15.0),
+ ... ],
+ ... columns=["name", "class", "max_speed"],
+ ... index=[4, 3, 2, 1, 0],
+ ... )
>>> df
name class max_speed
4 falcon bird 389.0
@@ -1038,7 +1031,7 @@ def take(
0 rabbit mammal 15.0
>>> gb = df["name"].groupby([1, 1, 2, 2, 2])
- Take elements at positions 0 and 1 along the axis 0 in each group (default).
+ Take elements at rows 0 and 1 in each group.
>>> gb.take([0, 1])
1 4 falcon
@@ -1057,12 +1050,11 @@ def take(
1 monkey
Name: name, dtype: object
"""
- result = self._op_via_apply("take", indices=indices, axis=axis, **kwargs)
+ result = self._op_via_apply("take", indices=indices, **kwargs)
return result
def skew(
self,
- axis: Axis | lib.NoDefault = lib.no_default,
skipna: bool = True,
numeric_only: bool = False,
**kwargs,
@@ -1074,14 +1066,6 @@ def skew(
Parameters
----------
- axis : {0 or 'index', 1 or 'columns', None}, default 0
- Axis for the function to be applied on.
- This parameter is only for compatibility with DataFrame and is unused.
-
- .. deprecated:: 2.1.0
- For axis=1, operate on the underlying object instead. Otherwise
- the axis keyword is not necessary.
-
skipna : bool, default True
Exclude NA/null values when computing the result.
@@ -1101,10 +1085,19 @@ def skew(
Examples
--------
- >>> ser = pd.Series([390., 350., 357., np.nan, 22., 20., 30.],
- ... index=['Falcon', 'Falcon', 'Falcon', 'Falcon',
- ... 'Parrot', 'Parrot', 'Parrot'],
- ... name="Max Speed")
+ >>> ser = pd.Series(
+ ... [390.0, 350.0, 357.0, np.nan, 22.0, 20.0, 30.0],
+ ... index=[
+ ... "Falcon",
+ ... "Falcon",
+ ... "Falcon",
+ ... "Falcon",
+ ... "Parrot",
+ ... "Parrot",
+ ... "Parrot",
+ ... ],
+ ... name="Max Speed",
+ ... )
>>> ser
Falcon 390.0
Falcon 350.0
@@ -1123,18 +1116,6 @@ def skew(
Parrot 1.457863
Name: Max Speed, dtype: float64
"""
- if axis is lib.no_default:
- axis = 0
-
- if axis != 0:
- result = self._op_via_apply(
- "skew",
- axis=axis,
- skipna=skipna,
- numeric_only=numeric_only,
- **kwargs,
- )
- return result
def alt(obj):
# This should not be reached since the cython path should raise
@@ -1173,17 +1154,111 @@ def nsmallest(
result = self._python_apply_general(f, data, not_indexed_same=True)
return result
- @doc(Series.idxmin.__doc__)
- def idxmin(
- self, axis: Axis | lib.NoDefault = lib.no_default, skipna: bool = True
- ) -> Series:
- return self._idxmax_idxmin("idxmin", axis=axis, skipna=skipna)
+ def idxmin(self, skipna: bool = True) -> Series:
+ """
+ Return the row label of the minimum value.
- @doc(Series.idxmax.__doc__)
- def idxmax(
- self, axis: Axis | lib.NoDefault = lib.no_default, skipna: bool = True
- ) -> Series:
- return self._idxmax_idxmin("idxmax", axis=axis, skipna=skipna)
+ If multiple values equal the minimum, the first row label with that
+ value is returned.
+
+ Parameters
+ ----------
+ skipna : bool, default True
+ Exclude NA values.
+
+ Returns
+ -------
+ Index
+ Label of the minimum value.
+
+ Raises
+ ------
+ ValueError
+ If the Series is empty or skipna=False and any value is NA.
+
+ See Also
+ --------
+ numpy.argmin : Return indices of the minimum values
+ along the given axis.
+ DataFrame.idxmin : Return index of first occurrence of minimum
+ over requested axis.
+ Series.idxmax : Return index *label* of the first occurrence
+ of maximum of values.
+
+ Examples
+ --------
+ >>> ser = pd.Series(
+ ... [1, 2, 3, 4],
+ ... index=pd.DatetimeIndex(
+ ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
+ ... ),
+ ... )
+ >>> ser
+ 2023-01-01 1
+ 2023-01-15 2
+ 2023-02-01 3
+ 2023-02-15 4
+ dtype: int64
+
+ >>> ser.groupby(["a", "a", "b", "b"]).idxmin()
+ a 2023-01-01
+ b 2023-02-01
+ dtype: datetime64[ns]
+ """
+ return self._idxmax_idxmin("idxmin", skipna=skipna)
+
+ def idxmax(self, skipna: bool = True) -> Series:
+ """
+ Return the row label of the maximum value.
+
+ If multiple values equal the maximum, the first row label with that
+ value is returned.
+
+ Parameters
+ ----------
+ skipna : bool, default True
+ Exclude NA values.
+
+ Returns
+ -------
+ Index
+ Label of the maximum value.
+
+ Raises
+ ------
+ ValueError
+ If the Series is empty or skipna=False and any value is NA.
+
+ See Also
+ --------
+ numpy.argmax : Return indices of the maximum values
+ along the given axis.
+ DataFrame.idxmax : Return index of first occurrence of maximum
+ over requested axis.
+ Series.idxmin : Return index *label* of the first occurrence
+ of minimum of values.
+
+ Examples
+ --------
+ >>> ser = pd.Series(
+ ... [1, 2, 3, 4],
+ ... index=pd.DatetimeIndex(
+ ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
+ ... ),
+ ... )
+ >>> ser
+ 2023-01-01 1
+ 2023-01-15 2
+ 2023-02-01 3
+ 2023-02-15 4
+ dtype: int64
+
+ >>> ser.groupby(["a", "a", "b", "b"]).idxmax()
+ a 2023-01-15
+ b 2023-02-15
+ dtype: datetime64[ns]
+ """
+ return self._idxmax_idxmin("idxmax", skipna=skipna)
@doc(Series.corr.__doc__)
def corr(
@@ -1217,7 +1292,7 @@ def is_monotonic_increasing(self) -> Series:
Examples
--------
- >>> s = pd.Series([2, 1, 3, 4], index=['Falcon', 'Falcon', 'Parrot', 'Parrot'])
+ >>> s = pd.Series([2, 1, 3, 4], index=["Falcon", "Falcon", "Parrot", "Parrot"])
>>> s.groupby(level=0).is_monotonic_increasing
Falcon False
Parrot True
@@ -1236,7 +1311,7 @@ def is_monotonic_decreasing(self) -> Series:
Examples
--------
- >>> s = pd.Series([2, 1, 3, 4], index=['Falcon', 'Falcon', 'Parrot', 'Parrot'])
+ >>> s = pd.Series([2, 1, 3, 4], index=["Falcon", "Falcon", "Parrot", "Parrot"])
>>> s.groupby(level=0).is_monotonic_decreasing
Falcon True
Parrot False
@@ -1254,7 +1329,7 @@ def hist(
xrot: float | None = None,
ylabelsize: int | None = None,
yrot: float | None = None,
- figsize: tuple[int, int] | None = None,
+ figsize: tuple[float, float] | None = None,
bins: int | Sequence[int] = 10,
backend: str | None = None,
legend: bool = False,
@@ -1300,13 +1375,17 @@ def unique(self) -> Series:
Examples
--------
- >>> df = pd.DataFrame([('Chihuahua', 'dog', 6.1),
- ... ('Beagle', 'dog', 15.2),
- ... ('Chihuahua', 'dog', 6.9),
- ... ('Persian', 'cat', 9.2),
- ... ('Chihuahua', 'dog', 7),
- ... ('Persian', 'cat', 8.8)],
- ... columns=['breed', 'animal', 'height_in'])
+ >>> df = pd.DataFrame(
+ ... [
+ ... ("Chihuahua", "dog", 6.1),
+ ... ("Beagle", "dog", 15.2),
+ ... ("Chihuahua", "dog", 6.9),
+ ... ("Persian", "cat", 9.2),
+ ... ("Chihuahua", "dog", 7),
+ ... ("Persian", "cat", 8.8),
+ ... ],
+ ... columns=["breed", "animal", "height_in"],
+ ... )
>>> df
breed animal height_in
0 Chihuahua dog 6.1
@@ -1315,7 +1394,7 @@ def unique(self) -> Series:
3 Persian cat 9.2
4 Chihuahua dog 7.0
5 Persian cat 8.8
- >>> ser = df.groupby('animal')['breed'].unique()
+ >>> ser = df.groupby("animal")["breed"].unique()
>>> ser
animal
cat [Persian]
@@ -1469,12 +1548,6 @@ def aggregate(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs)
# can't return early
result = self._aggregate_frame(func, *args, **kwargs)
- elif self.axis == 1:
- # _aggregate_multiple_funcs does not allow self.axis == 1
- # Note: axis == 1 precludes 'not self.as_index', see __init__
- result = self._aggregate_frame(func)
- return result
-
else:
# try to treat as if we are passing a list
gba = GroupByApply(self, [func], args=(), kwargs={})
@@ -1507,11 +1580,6 @@ def aggregate(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs)
agg = aggregate
def _python_agg_general(self, func, *args, **kwargs):
- orig_func = func
- func = com.is_builtin_func(func)
- if orig_func != func:
- alias = com._builtin_table_alias[func]
- warn_alias_replacement(self, orig_func, alias)
f = lambda x: func(x, *args, **kwargs)
if self.ngroups == 0:
@@ -1520,8 +1588,6 @@ def _python_agg_general(self, func, *args, **kwargs):
return self._python_apply_general(f, self._selected_obj, is_agg=True)
obj = self._obj_with_exclusions
- if self.axis == 1:
- obj = obj.T
if not len(obj.columns):
# e.g. test_margins_no_values_no_cols
@@ -1543,15 +1609,13 @@ def _aggregate_frame(self, func, *args, **kwargs) -> DataFrame:
obj = self._obj_with_exclusions
result: dict[Hashable, NDFrame | np.ndarray] = {}
- for name, grp_df in self._grouper.get_iterator(obj, self.axis):
+ for name, grp_df in self._grouper.get_iterator(obj):
fres = func(grp_df, *args, **kwargs)
result[name] = fres
result_index = self._grouper.result_index
- other_ax = obj.axes[1 - self.axis]
- out = self.obj._constructor(result, index=other_ax, columns=result_index)
- if self.axis == 0:
- out = out.T
+ out = self.obj._constructor(result, index=obj.columns, columns=result_index)
+ out = out.T
return out
@@ -1570,7 +1634,7 @@ def _wrap_applied_output(
res_index = self._grouper.result_index
result = self.obj._constructor(index=res_index, columns=data.columns)
- result = result.astype(data.dtypes, copy=False)
+ result = result.astype(data.dtypes)
return result
# GH12824
@@ -1578,8 +1642,11 @@ def _wrap_applied_output(
first_not_none = next(com.not_none(*values), None)
if first_not_none is None:
- # GH9684 - All values are None, return an empty frame.
- return self.obj._constructor()
+ # GH9684 - All values are None, return an empty frame
+ # GH57775 - Ensure that columns and dtypes from original frame are kept.
+ result = self.obj._constructor(columns=data.columns)
+ result = result.astype(data.dtypes)
+ return result
elif isinstance(first_not_none, DataFrame):
return self._concat_objects(
values,
@@ -1651,18 +1718,13 @@ def _wrap_applied_output_series(
# vstack+constructor is faster than concat and handles MI-columns
stacked_values = np.vstack([np.asarray(v) for v in values])
- if self.axis == 0:
- index = key_index
- columns = first_not_none.index.copy()
- if columns.name is None:
- # GH6124 - propagate name of Series when it's consistent
- names = {v.name for v in values}
- if len(names) == 1:
- columns.name = next(iter(names))
- else:
- index = first_not_none.index
- columns = key_index
- stacked_values = stacked_values.T
+ index = key_index
+ columns = first_not_none.index.copy()
+ if columns.name is None:
+ # GH6124 - propagate name of Series when it's consistent
+ names = {v.name for v in values}
+ if len(names) == 1:
+ columns.name = next(iter(names))
if stacked_values.dtype == object:
# We'll have the DataFrame constructor do inference
@@ -1672,23 +1734,18 @@ def _wrap_applied_output_series(
if not self.as_index:
result = self._insert_inaxis_grouper(result)
- return self._reindex_output(result)
+ return result
def _cython_transform(
self,
how: str,
numeric_only: bool = False,
- axis: AxisInt = 0,
**kwargs,
) -> DataFrame:
- assert axis == 0 # handled by caller
-
- # With self.axis == 0, we have multi-block tests
+ # We have multi-block tests
# e.g. test_rank_min_int, test_cython_transform_frame
# test_transform_numeric_ret
- # With self.axis == 1, _get_data_to_aggregate does a transpose
- # so we always have a single block.
- mgr: Manager2D = self._get_data_to_aggregate(
+ mgr: BlockManager = self._get_data_to_aggregate(
numeric_only=numeric_only, name=how
)
@@ -1697,13 +1754,9 @@ def arr_func(bvalues: ArrayLike) -> ArrayLike:
"transform", bvalues, how, 1, **kwargs
)
- # We could use `mgr.apply` here and not have to set_axis, but
- # we would have to do shape gymnastics for ArrayManager compat
- res_mgr = mgr.grouped_reduce(arr_func)
- res_mgr.set_axis(1, mgr.axes[1])
+ res_mgr = mgr.apply(arr_func)
res_df = self.obj._constructor_from_mgr(res_mgr, axes=res_mgr.axes)
- res_df = self._maybe_transpose_result(res_df)
return res_df
def _transform_general(self, func, engine, engine_kwargs, *args, **kwargs):
@@ -1715,7 +1768,7 @@ def _transform_general(self, func, engine, engine_kwargs, *args, **kwargs):
applied = []
obj = self._obj_with_exclusions
- gen = self._grouper.get_iterator(obj, axis=self.axis)
+ gen = self._grouper.get_iterator(obj)
fast_path, slow_path = self._define_paths(func, *args, **kwargs)
# Determine whether to use slow or fast path by evaluating on the first group.
@@ -1749,10 +1802,11 @@ def _transform_general(self, func, engine, engine_kwargs, *args, **kwargs):
res = _wrap_transform_general_frame(self.obj, group, res)
applied.append(res)
- concat_index = obj.columns if self.axis == 0 else obj.index
- other_axis = 1 if self.axis == 0 else 0 # switches between 0 & 1
- concatenated = concat(applied, axis=self.axis, verify_integrity=False)
- concatenated = concatenated.reindex(concat_index, axis=other_axis, copy=False)
+ concat_index = obj.columns
+ concatenated = concat(
+ applied, axis=0, verify_integrity=False, ignore_index=True
+ )
+ concatenated = concatenated.reindex(concat_index, axis=1)
return self._set_result_index_ordered(concatenated)
__examples_dataframe_doc = dedent(
@@ -1820,12 +1874,12 @@ def _define_paths(self, func, *args, **kwargs):
if isinstance(func, str):
fast_path = lambda group: getattr(group, func)(*args, **kwargs)
slow_path = lambda group: group.apply(
- lambda x: getattr(x, func)(*args, **kwargs), axis=self.axis
+ lambda x: getattr(x, func)(*args, **kwargs), axis=0
)
else:
fast_path = lambda group: func(group, *args, **kwargs)
slow_path = lambda group: group.apply(
- lambda x: func(x, *args, **kwargs), axis=self.axis
+ lambda x: func(x, *args, **kwargs), axis=0
)
return fast_path, slow_path
@@ -1865,7 +1919,7 @@ def _choose_path(self, fast_path: Callable, slow_path: Callable, group: DataFram
return path, res
- def filter(self, func, dropna: bool = True, *args, **kwargs):
+ def filter(self, func, dropna: bool = True, *args, **kwargs) -> DataFrame:
"""
Filter elements from groups that don't satisfy a criterion.
@@ -1879,6 +1933,10 @@ def filter(self, func, dropna: bool = True, *args, **kwargs):
dropna : bool
Drop groups that do not pass the filter. True by default; if False,
groups that evaluate False are filled with NaNs.
+ *args
+ Additional positional arguments to pass to `func`.
+ **kwargs
+ Additional keyword arguments to pass to `func`.
Returns
-------
@@ -1895,12 +1953,15 @@ def filter(self, func, dropna: bool = True, *args, **kwargs):
Examples
--------
- >>> df = pd.DataFrame({'A' : ['foo', 'bar', 'foo', 'bar',
- ... 'foo', 'bar'],
- ... 'B' : [1, 2, 3, 4, 5, 6],
- ... 'C' : [2.0, 5., 8., 1., 2., 9.]})
- >>> grouped = df.groupby('A')
- >>> grouped.filter(lambda x: x['B'].mean() > 3.)
+ >>> df = pd.DataFrame(
+ ... {
+ ... "A": ["foo", "bar", "foo", "bar", "foo", "bar"],
+ ... "B": [1, 2, 3, 4, 5, 6],
+ ... "C": [2.0, 5.0, 8.0, 1.0, 2.0, 9.0],
+ ... }
+ ... )
+ >>> grouped = df.groupby("A")
+ >>> grouped.filter(lambda x: x["B"].mean() > 3.0)
A B C
1 bar 2 5.0
3 bar 4 1.0
@@ -1909,7 +1970,7 @@ def filter(self, func, dropna: bool = True, *args, **kwargs):
indices = []
obj = self._selected_obj
- gen = self._grouper.get_iterator(obj, axis=self.axis)
+ gen = self._grouper.get_iterator(obj)
for name, group in gen:
# 2023-02-27 no tests are broken this pinning, but it is documented in the
@@ -1937,9 +1998,6 @@ def filter(self, func, dropna: bool = True, *args, **kwargs):
return self._apply_filter(indices, dropna)
def __getitem__(self, key) -> DataFrameGroupBy | SeriesGroupBy:
- if self.axis == 1:
- # GH 37725
- raise ValueError("Cannot subset columns when using axis=1")
# per GH 23566
if isinstance(key, tuple) and len(key) > 1:
# if len == 1, then it becomes a SeriesGroupBy and this is actually
@@ -1969,7 +2027,6 @@ def _gotitem(self, key, ndim: int, subset=None):
return DataFrameGroupBy(
subset,
self.keys,
- axis=self.axis,
level=self.level,
grouper=self._grouper,
exclusions=self.exclusions,
@@ -2001,18 +2058,14 @@ def _gotitem(self, key, ndim: int, subset=None):
def _get_data_to_aggregate(
self, *, numeric_only: bool = False, name: str | None = None
- ) -> Manager2D:
+ ) -> BlockManager:
obj = self._obj_with_exclusions
- if self.axis == 1:
- mgr = obj.T._mgr
- else:
- mgr = obj._mgr
-
+ mgr = obj._mgr
if numeric_only:
mgr = mgr.get_numeric_data()
return mgr
- def _wrap_agged_manager(self, mgr: Manager2D) -> DataFrame:
+ def _wrap_agged_manager(self, mgr: BlockManager) -> DataFrame:
return self.obj._constructor_from_mgr(mgr, axes=mgr.axes)
def _apply_to_column_groupbys(self, func) -> DataFrame:
@@ -2058,10 +2111,13 @@ def nunique(self, dropna: bool = True) -> DataFrame:
Examples
--------
- >>> df = pd.DataFrame({'id': ['spam', 'egg', 'egg', 'spam',
- ... 'ham', 'ham'],
- ... 'value1': [1, 5, 5, 2, 5, 5],
- ... 'value2': list('abbaxy')})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "id": ["spam", "egg", "egg", "spam", "ham", "ham"],
+ ... "value1": [1, 5, 5, 2, 5, 5],
+ ... "value2": list("abbaxy"),
+ ... }
+ ... )
>>> df
id value1 value2
0 spam 1 a
@@ -2071,7 +2127,7 @@ def nunique(self, dropna: bool = True) -> DataFrame:
4 ham 5 x
5 ham 5 y
- >>> df.groupby('id').nunique()
+ >>> df.groupby("id").nunique()
value1 value2
id
egg 1 1
@@ -2080,48 +2136,27 @@ def nunique(self, dropna: bool = True) -> DataFrame:
Check for rows with the same id but conflicting values:
- >>> df.groupby('id').filter(lambda g: (g.nunique() > 1).any())
+ >>> df.groupby("id").filter(lambda g: (g.nunique() > 1).any())
id value1 value2
0 spam 1 a
3 spam 2 a
4 ham 5 x
5 ham 5 y
"""
-
- if self.axis != 0:
- # see test_groupby_crash_on_nunique
- return self._python_apply_general(
- lambda sgb: sgb.nunique(dropna), self._obj_with_exclusions, is_agg=True
- )
-
return self._apply_to_column_groupbys(lambda sgb: sgb.nunique(dropna))
def idxmax(
self,
- axis: Axis | None | lib.NoDefault = lib.no_default,
skipna: bool = True,
numeric_only: bool = False,
) -> DataFrame:
"""
- Return index of first occurrence of maximum over requested axis.
-
- NA/null values are excluded.
+ Return index of first occurrence of maximum in each group.
Parameters
----------
- axis : {{0 or 'index', 1 or 'columns'}}, default None
- The axis to use. 0 or 'index' for row-wise, 1 or 'columns' for column-wise.
- If axis is not provided, grouper's axis is used.
-
- .. versionchanged:: 2.0.0
-
- .. deprecated:: 2.1.0
- For axis=1, operate on the underlying object instead. Otherwise
- the axis keyword is not necessary.
-
skipna : bool, default True
- Exclude NA/null values. If an entire row/column is NA, the result
- will be NA.
+ Exclude NA values.
numeric_only : bool, default False
Include only `float`, `int` or `boolean` data.
@@ -2130,12 +2165,12 @@ def idxmax(
Returns
-------
Series
- Indexes of maxima along the specified axis.
+ Indexes of maxima in each group.
Raises
------
ValueError
- * If the row/column is empty
+ * If a column is empty or skipna=False and any value is NA.
See Also
--------
@@ -2149,9 +2184,13 @@ def idxmax(
--------
Consider a dataset containing food consumption in Argentina.
- >>> df = pd.DataFrame({'consumption': [10.51, 103.11, 55.48],
- ... 'co2_emissions': [37.2, 19.66, 1712]},
- ... index=['Pork', 'Wheat Products', 'Beef'])
+ >>> df = pd.DataFrame(
+ ... {
+ ... "consumption": [10.51, 103.11, 55.48],
+ ... "co2_emissions": [37.2, 19.66, 1712],
+ ... },
+ ... index=["Pork", "Wheat Products", "Beef"],
+ ... )
>>> df
consumption co2_emissions
@@ -2165,45 +2204,21 @@ def idxmax(
consumption Wheat Products
co2_emissions Beef
dtype: object
-
- To return the index for the maximum value in each row, use ``axis="columns"``.
-
- >>> df.idxmax(axis="columns")
- Pork co2_emissions
- Wheat Products consumption
- Beef co2_emissions
- dtype: object
"""
- return self._idxmax_idxmin(
- "idxmax", axis=axis, numeric_only=numeric_only, skipna=skipna
- )
+ return self._idxmax_idxmin("idxmax", numeric_only=numeric_only, skipna=skipna)
def idxmin(
self,
- axis: Axis | None | lib.NoDefault = lib.no_default,
skipna: bool = True,
numeric_only: bool = False,
) -> DataFrame:
"""
- Return index of first occurrence of minimum over requested axis.
-
- NA/null values are excluded.
+ Return index of first occurrence of minimum in each group.
Parameters
----------
- axis : {{0 or 'index', 1 or 'columns'}}, default None
- The axis to use. 0 or 'index' for row-wise, 1 or 'columns' for column-wise.
- If axis is not provided, grouper's axis is used.
-
- .. versionchanged:: 2.0.0
-
- .. deprecated:: 2.1.0
- For axis=1, operate on the underlying object instead. Otherwise
- the axis keyword is not necessary.
-
skipna : bool, default True
- Exclude NA/null values. If an entire row/column is NA, the result
- will be NA.
+ Exclude NA values.
numeric_only : bool, default False
Include only `float`, `int` or `boolean` data.
@@ -2212,12 +2227,12 @@ def idxmin(
Returns
-------
Series
- Indexes of minima along the specified axis.
+ Indexes of minima in each group.
Raises
------
ValueError
- * If the row/column is empty
+ * If a column is empty or skipna=False and any value is NA.
See Also
--------
@@ -2231,9 +2246,13 @@ def idxmin(
--------
Consider a dataset containing food consumption in Argentina.
- >>> df = pd.DataFrame({'consumption': [10.51, 103.11, 55.48],
- ... 'co2_emissions': [37.2, 19.66, 1712]},
- ... index=['Pork', 'Wheat Products', 'Beef'])
+ >>> df = pd.DataFrame(
+ ... {
+ ... "consumption": [10.51, 103.11, 55.48],
+ ... "co2_emissions": [37.2, 19.66, 1712],
+ ... },
+ ... index=["Pork", "Wheat Products", "Beef"],
+ ... )
>>> df
consumption co2_emissions
@@ -2247,18 +2266,8 @@ def idxmin(
consumption Pork
co2_emissions Wheat Products
dtype: object
-
- To return the index for the minimum value in each row, use ``axis="columns"``.
-
- >>> df.idxmin(axis="columns")
- Pork consumption
- Wheat Products co2_emissions
- Beef consumption
- dtype: object
"""
- return self._idxmax_idxmin(
- "idxmin", axis=axis, numeric_only=numeric_only, skipna=skipna
- )
+ return self._idxmax_idxmin("idxmin", numeric_only=numeric_only, skipna=skipna)
boxplot = boxplot_frame_groupby
@@ -2291,7 +2300,7 @@ def value_counts(
Returns
-------
Series or DataFrame
- Series if the groupby as_index is True, otherwise DataFrame.
+ Series if the groupby ``as_index`` is True, otherwise DataFrame.
See Also
--------
@@ -2301,9 +2310,9 @@ def value_counts(
Notes
-----
- - If the groupby as_index is True then the returned Series will have a
+ - If the groupby ``as_index`` is True then the returned Series will have a
MultiIndex with one level per input column.
- - If the groupby as_index is False then the returned DataFrame will have an
+ - If the groupby ``as_index`` is False then the returned DataFrame will have an
additional column with the value_counts. The column is labelled 'count' or
'proportion', depending on the ``normalize`` parameter.
@@ -2315,11 +2324,13 @@ def value_counts(
Examples
--------
- >>> df = pd.DataFrame({
- ... 'gender': ['male', 'male', 'female', 'male', 'female', 'male'],
- ... 'education': ['low', 'medium', 'high', 'low', 'high', 'low'],
- ... 'country': ['US', 'FR', 'US', 'FR', 'FR', 'FR']
- ... })
+ >>> df = pd.DataFrame(
+ ... {
+ ... "gender": ["male", "male", "female", "male", "female", "male"],
+ ... "education": ["low", "medium", "high", "low", "high", "low"],
+ ... "country": ["US", "FR", "US", "FR", "FR", "FR"],
+ ... }
+ ... )
>>> df
gender education country
@@ -2330,7 +2341,7 @@ def value_counts(
4 female high FR
5 male low FR
- >>> df.groupby('gender').value_counts()
+ >>> df.groupby("gender").value_counts()
gender education country
female high FR 1
US 1
@@ -2339,7 +2350,7 @@ def value_counts(
medium FR 1
Name: count, dtype: int64
- >>> df.groupby('gender').value_counts(ascending=True)
+ >>> df.groupby("gender").value_counts(ascending=True)
gender education country
female high FR 1
US 1
@@ -2348,7 +2359,7 @@ def value_counts(
low FR 2
Name: count, dtype: int64
- >>> df.groupby('gender').value_counts(normalize=True)
+ >>> df.groupby("gender").value_counts(normalize=True)
gender education country
female high FR 0.50
US 0.50
@@ -2357,7 +2368,7 @@ def value_counts(
medium FR 0.25
Name: proportion, dtype: float64
- >>> df.groupby('gender', as_index=False).value_counts()
+ >>> df.groupby("gender", as_index=False).value_counts()
gender education country count
0 female high FR 1
1 female high US 1
@@ -2365,7 +2376,7 @@ def value_counts(
3 male low US 1
4 male medium FR 1
- >>> df.groupby('gender', as_index=False).value_counts(normalize=True)
+ >>> df.groupby("gender", as_index=False).value_counts(normalize=True)
gender education country proportion
0 female high FR 0.50
1 female high US 0.50
@@ -2375,156 +2386,9 @@ def value_counts(
"""
return self._value_counts(subset, normalize, sort, ascending, dropna)
- def fillna(
- self,
- value: Hashable | Mapping | Series | DataFrame | None = None,
- method: FillnaOptions | None = None,
- axis: Axis | None | lib.NoDefault = lib.no_default,
- inplace: bool = False,
- limit: int | None = None,
- downcast=lib.no_default,
- ) -> DataFrame | None:
- """
- Fill NA/NaN values using the specified method within groups.
-
- .. deprecated:: 2.2.0
- This method is deprecated and will be removed in a future version.
- Use the :meth:`.DataFrameGroupBy.ffill` or :meth:`.DataFrameGroupBy.bfill`
- for forward or backward filling instead. If you want to fill with a
- single value, use :meth:`DataFrame.fillna` instead.
-
- Parameters
- ----------
- value : scalar, dict, Series, or DataFrame
- Value to use to fill holes (e.g. 0), alternately a
- dict/Series/DataFrame of values specifying which value to use for
- each index (for a Series) or column (for a DataFrame). Values not
- in the dict/Series/DataFrame will not be filled. This value cannot
- be a list. Users wanting to use the ``value`` argument and not ``method``
- should prefer :meth:`.DataFrame.fillna` as this
- will produce the same result and be more performant.
- method : {{'bfill', 'ffill', None}}, default None
- Method to use for filling holes. ``'ffill'`` will propagate
- the last valid observation forward within a group.
- ``'bfill'`` will use next valid observation to fill the gap.
- axis : {0 or 'index', 1 or 'columns'}
- Axis along which to fill missing values. When the :class:`DataFrameGroupBy`
- ``axis`` argument is ``0``, using ``axis=1`` here will produce
- the same results as :meth:`.DataFrame.fillna`. When the
- :class:`DataFrameGroupBy` ``axis`` argument is ``1``, using ``axis=0``
- or ``axis=1`` here will produce the same results.
- inplace : bool, default False
- Broken. Do not set to True.
- limit : int, default None
- If method is specified, this is the maximum number of consecutive
- NaN values to forward/backward fill within a group. In other words,
- if there is a gap with more than this number of consecutive NaNs,
- it will only be partially filled. If method is not specified, this is the
- maximum number of entries along the entire axis where NaNs will be
- filled. Must be greater than 0 if not None.
- downcast : dict, default is None
- A dict of item->dtype of what to downcast if possible,
- or the string 'infer' which will try to downcast to an appropriate
- equal type (e.g. float64 to int64 if possible).
-
- Returns
- -------
- DataFrame
- Object with missing values filled.
-
- See Also
- --------
- ffill : Forward fill values within a group.
- bfill : Backward fill values within a group.
-
- Examples
- --------
- >>> df = pd.DataFrame(
- ... {
- ... "key": [0, 0, 1, 1, 1],
- ... "A": [np.nan, 2, np.nan, 3, np.nan],
- ... "B": [2, 3, np.nan, np.nan, np.nan],
- ... "C": [np.nan, np.nan, 2, np.nan, np.nan],
- ... }
- ... )
- >>> df
- key A B C
- 0 0 NaN 2.0 NaN
- 1 0 2.0 3.0 NaN
- 2 1 NaN NaN 2.0
- 3 1 3.0 NaN NaN
- 4 1 NaN NaN NaN
-
- Propagate non-null values forward or backward within each group along columns.
-
- >>> df.groupby("key").fillna(method="ffill")
- A B C
- 0 NaN 2.0 NaN
- 1 2.0 3.0 NaN
- 2 NaN NaN 2.0
- 3 3.0 NaN 2.0
- 4 3.0 NaN 2.0
-
- >>> df.groupby("key").fillna(method="bfill")
- A B C
- 0 2.0 2.0 NaN
- 1 2.0 3.0 NaN
- 2 3.0 NaN 2.0
- 3 3.0 NaN NaN
- 4 NaN NaN NaN
-
- Propagate non-null values forward or backward within each group along rows.
-
- >>> df.T.groupby(np.array([0, 0, 1, 1])).fillna(method="ffill").T
- key A B C
- 0 0.0 0.0 2.0 2.0
- 1 0.0 2.0 3.0 3.0
- 2 1.0 1.0 NaN 2.0
- 3 1.0 3.0 NaN NaN
- 4 1.0 1.0 NaN NaN
-
- >>> df.T.groupby(np.array([0, 0, 1, 1])).fillna(method="bfill").T
- key A B C
- 0 0.0 NaN 2.0 NaN
- 1 0.0 2.0 3.0 NaN
- 2 1.0 NaN 2.0 2.0
- 3 1.0 3.0 NaN NaN
- 4 1.0 NaN NaN NaN
-
- Only replace the first NaN element within a group along rows.
-
- >>> df.groupby("key").fillna(method="ffill", limit=1)
- A B C
- 0 NaN 2.0 NaN
- 1 2.0 3.0 NaN
- 2 NaN NaN 2.0
- 3 3.0 NaN 2.0
- 4 3.0 NaN NaN
- """
- warnings.warn(
- f"{type(self).__name__}.fillna is deprecated and "
- "will be removed in a future version. Use obj.ffill() or obj.bfill() "
- "for forward or backward filling instead. If you want to fill with a "
- f"single value, use {type(self.obj).__name__}.fillna instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- result = self._op_via_apply(
- "fillna",
- value=value,
- method=method,
- axis=axis,
- inplace=inplace,
- limit=limit,
- downcast=downcast,
- )
- return result
-
def take(
self,
indices: TakeIndexer,
- axis: Axis | None | lib.NoDefault = lib.no_default,
**kwargs,
) -> DataFrame:
"""
@@ -2542,13 +2406,6 @@ def take(
----------
indices : array-like
An array of ints indicating which positions to take.
- axis : {0 or 'index', 1 or 'columns', None}, default 0
- The axis on which to select elements. ``0`` means that we are
- selecting rows, ``1`` means that we are selecting columns.
-
- .. deprecated:: 2.1.0
- For axis=1, operate on the underlying object instead. Otherwise
- the axis keyword is not necessary.
**kwargs
For compatibility with :meth:`numpy.take`. Has no effect on the
@@ -2568,13 +2425,17 @@ def take(
Examples
--------
- >>> df = pd.DataFrame([('falcon', 'bird', 389.0),
- ... ('parrot', 'bird', 24.0),
- ... ('lion', 'mammal', 80.5),
- ... ('monkey', 'mammal', np.nan),
- ... ('rabbit', 'mammal', 15.0)],
- ... columns=['name', 'class', 'max_speed'],
- ... index=[4, 3, 2, 1, 0])
+ >>> df = pd.DataFrame(
+ ... [
+ ... ("falcon", "bird", 389.0),
+ ... ("parrot", "bird", 24.0),
+ ... ("lion", "mammal", 80.5),
+ ... ("monkey", "mammal", np.nan),
+ ... ("rabbit", "mammal", 15.0),
+ ... ],
+ ... columns=["name", "class", "max_speed"],
+ ... index=[4, 3, 2, 1, 0],
+ ... )
>>> df
name class max_speed
4 falcon bird 389.0
@@ -2584,7 +2445,7 @@ def take(
0 rabbit mammal 15.0
>>> gb = df.groupby([1, 1, 2, 2, 2])
- Take elements at positions 0 and 1 along the axis 0 (default).
+ Take elements at rows 0 and 1.
Note how the indices selected in the result do not correspond to
our input indices 0 and 1. That's because we are selecting the 0th
@@ -2607,8 +2468,6 @@ def take(
2 1 monkey mammal NaN
2 lion mammal 80.5
- Take elements at indices 1 and 2 along the axis 1 (column selection).
-
We may take elements using negative integers for positive indices,
starting from the end of the object, just like with Python lists.
@@ -2619,12 +2478,11 @@ def take(
2 0 rabbit mammal 15.0
1 monkey mammal NaN
"""
- result = self._op_via_apply("take", indices=indices, axis=axis, **kwargs)
+ result = self._op_via_apply("take", indices=indices, **kwargs)
return result
def skew(
self,
- axis: Axis | None | lib.NoDefault = lib.no_default,
skipna: bool = True,
numeric_only: bool = False,
**kwargs,
@@ -2636,17 +2494,6 @@ def skew(
Parameters
----------
- axis : {0 or 'index', 1 or 'columns', None}, default 0
- Axis for the function to be applied on.
-
- Specifying ``axis=None`` will apply the aggregation across both axes.
-
- .. versionadded:: 2.0.0
-
- .. deprecated:: 2.1.0
- For axis=1, operate on the underlying object instead. Otherwise
- the axis keyword is not necessary.
-
skipna : bool, default True
Exclude NA/null values when computing the result.
@@ -2666,14 +2513,15 @@ def skew(
Examples
--------
- >>> arrays = [['falcon', 'parrot', 'cockatoo', 'kiwi',
- ... 'lion', 'monkey', 'rabbit'],
- ... ['bird', 'bird', 'bird', 'bird',
- ... 'mammal', 'mammal', 'mammal']]
- >>> index = pd.MultiIndex.from_arrays(arrays, names=('name', 'class'))
- >>> df = pd.DataFrame({'max_speed': [389.0, 24.0, 70.0, np.nan,
- ... 80.5, 21.5, 15.0]},
- ... index=index)
+ >>> arrays = [
+ ... ["falcon", "parrot", "cockatoo", "kiwi", "lion", "monkey", "rabbit"],
+ ... ["bird", "bird", "bird", "bird", "mammal", "mammal", "mammal"],
+ ... ]
+ >>> index = pd.MultiIndex.from_arrays(arrays, names=("name", "class"))
+ >>> df = pd.DataFrame(
+ ... {"max_speed": [389.0, 24.0, 70.0, np.nan, 80.5, 21.5, 15.0]},
+ ... index=index,
+ ... )
>>> df
max_speed
name class
@@ -2696,18 +2544,6 @@ def skew(
bird NaN
mammal 1.669046
"""
- if axis is lib.no_default:
- axis = 0
-
- if axis != 0:
- result = self._op_via_apply(
- "skew",
- axis=axis,
- skipna=skipna,
- numeric_only=numeric_only,
- **kwargs,
- )
- return result
def alt(obj):
# This should not be reached since the cython path should raise
@@ -2748,7 +2584,6 @@ def cov(
)
return result
- @doc(DataFrame.hist.__doc__)
def hist(
self,
column: IndexLabel | None = None,
@@ -2761,13 +2596,99 @@ def hist(
ax=None,
sharex: bool = False,
sharey: bool = False,
- figsize: tuple[int, int] | None = None,
+ figsize: tuple[float, float] | None = None,
layout: tuple[int, int] | None = None,
bins: int | Sequence[int] = 10,
backend: str | None = None,
legend: bool = False,
**kwargs,
):
+ """
+ Make a histogram of the DataFrame's columns.
+
+ A `histogram`_ is a representation of the distribution of data.
+ This function calls :meth:`matplotlib.pyplot.hist`, on each series in
+ the DataFrame, resulting in one histogram per column.
+
+ .. _histogram: https://en.wikipedia.org/wiki/Histogram
+
+ Parameters
+ ----------
+ column : str or sequence, optional
+ If passed, will be used to limit data to a subset of columns.
+ by : object, optional
+ If passed, then used to form histograms for separate groups.
+ grid : bool, default True
+ Whether to show axis grid lines.
+ xlabelsize : int, default None
+ If specified changes the x-axis label size.
+ xrot : float, default None
+ Rotation of x axis labels. For example, a value of 90 displays the
+ x labels rotated 90 degrees clockwise.
+ ylabelsize : int, default None
+ If specified changes the y-axis label size.
+ yrot : float, default None
+ Rotation of y axis labels. For example, a value of 90 displays the
+ y labels rotated 90 degrees clockwise.
+ ax : Matplotlib axes object, default None
+ The axes to plot the histogram on.
+ sharex : bool, default True if ax is None else False
+ In case subplots=True, share x axis and set some x axis labels to
+ invisible; defaults to True if ax is None otherwise False if an ax
+ is passed in.
+ Note that passing in both an ax and sharex=True will alter all x axis
+ labels for all subplots in a figure.
+ sharey : bool, default False
+ In case subplots=True, share y axis and set some y axis labels to
+ invisible.
+ figsize : tuple, optional
+ The size in inches of the figure to create. Uses the value in
+ `matplotlib.rcParams` by default.
+ layout : tuple, optional
+ Tuple of (rows, columns) for the layout of the histograms.
+ bins : int or sequence, default 10
+ Number of histogram bins to be used. If an integer is given, bins + 1
+ bin edges are calculated and returned. If bins is a sequence, gives
+ bin edges, including left edge of first bin and right edge of last
+ bin. In this case, bins is returned unmodified.
+
+ backend : str, default None
+ Backend to use instead of the backend specified in the option
+ ``plotting.backend``. For instance, 'matplotlib'. Alternatively, to
+ specify the ``plotting.backend`` for the whole session, set
+ ``pd.options.plotting.backend``.
+
+ legend : bool, default False
+ Whether to show the legend.
+
+ **kwargs
+ All other plotting keyword arguments to be passed to
+ :meth:`matplotlib.pyplot.hist`.
+
+ Returns
+ -------
+ matplotlib.Axes or numpy.ndarray of them
+
+ See Also
+ --------
+ matplotlib.pyplot.hist : Plot a histogram using matplotlib.
+
+ Examples
+ --------
+ This example draws a histogram based on the length and width of
+ some animals, displayed in three bins
+
+ .. plot::
+ :context: close-figs
+
+ >>> data = {
+ ... "length": [1.5, 0.5, 1.2, 0.9, 3],
+ ... "width": [0.7, 0.2, 0.15, 0.2, 1.1],
+ ... }
+ >>> index = ["pig", "rabbit", "duck", "chicken", "horse"]
+ >>> df = pd.DataFrame(data, index=index)
+ >>> hist = df.groupby("length").hist(bins=3)
+ """
result = self._op_via_apply(
"hist",
column=column,
@@ -2789,35 +2710,78 @@ def hist(
)
return result
- @property
- @doc(DataFrame.dtypes.__doc__)
- def dtypes(self) -> Series:
- # GH#51045
- warnings.warn(
- f"{type(self).__name__}.dtypes is deprecated and will be removed in "
- "a future version. Check the dtypes on the base object instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- # error: Incompatible return value type (got "DataFrame", expected "Series")
- return self._python_apply_general( # type: ignore[return-value]
- lambda df: df.dtypes, self._selected_obj
- )
-
- @doc(DataFrame.corrwith.__doc__)
def corrwith(
self,
other: DataFrame | Series,
- axis: Axis | lib.NoDefault = lib.no_default,
drop: bool = False,
method: CorrelationMethod = "pearson",
numeric_only: bool = False,
) -> DataFrame:
+ """
+ Compute pairwise correlation.
+
+ Pairwise correlation is computed between rows or columns of
+ DataFrame with rows or columns of Series or DataFrame. DataFrames
+ are first aligned along both axes before computing the
+ correlations.
+
+ Parameters
+ ----------
+ other : DataFrame, Series
+ Object with which to compute correlations.
+ drop : bool, default False
+ Drop missing indices from result.
+ method : {'pearson', 'kendall', 'spearman'} or callable
+ Method of correlation:
+
+ * pearson : standard correlation coefficient
+ * kendall : Kendall Tau correlation coefficient
+ * spearman : Spearman rank correlation
+ * callable: callable with input two 1d ndarrays
+ and returning a float.
+
+ numeric_only : bool, default False
+ Include only `float`, `int` or `boolean` data.
+
+ .. versionadded:: 1.5.0
+
+ .. versionchanged:: 2.0.0
+ The default value of ``numeric_only`` is now ``False``.
+
+ Returns
+ -------
+ Series
+ Pairwise correlations.
+
+ See Also
+ --------
+ DataFrame.corr : Compute pairwise correlation of columns.
+
+ Examples
+ --------
+ >>> df1 = pd.DataFrame(
+ ... {
+ ... "Day": [1, 1, 1, 2, 2, 2, 3, 3, 3],
+ ... "Data": [6, 6, 8, 5, 4, 2, 7, 3, 9],
+ ... }
+ ... )
+ >>> df2 = pd.DataFrame(
+ ... {
+ ... "Day": [1, 1, 1, 2, 2, 2, 3, 3, 3],
+ ... "Data": [5, 3, 8, 3, 1, 1, 2, 3, 6],
+ ... }
+ ... )
+
+ >>> df1.groupby("Day").corrwith(df2)
+ Data Day
+ Day
+ 1 0.917663 NaN
+ 2 0.755929 NaN
+ 3 0.576557 NaN
+ """
result = self._op_via_apply(
"corrwith",
other=other,
- axis=axis,
drop=drop,
method=method,
numeric_only=numeric_only,
@@ -2835,7 +2799,7 @@ def _wrap_transform_general_frame(
# other dimension; this will preserve dtypes
# GH14457
if res.index.is_(obj.index):
- res_frame = concat([res] * len(group.columns), axis=1)
+ res_frame = concat([res] * len(group.columns), axis=1, ignore_index=True)
res_frame.columns = group.columns
res_frame.index = group.index
else:
diff --git a/pandas/core/groupby/groupby.py b/pandas/core/groupby/groupby.py
index 089e15afd465b..bd8e222831d0c 100644
--- a/pandas/core/groupby/groupby.py
+++ b/pandas/core/groupby/groupby.py
@@ -6,6 +6,7 @@ class providing the base-class of operations.
(defined in pandas.core.groupby.generic)
expose these user-facing objects to provide specific functionality.
"""
+
from __future__ import annotations
from collections.abc import (
@@ -19,7 +20,6 @@ class providing the base-class of operations.
partial,
wraps,
)
-import inspect
from textwrap import dedent
from typing import (
TYPE_CHECKING,
@@ -29,13 +29,12 @@ class providing the base-class of operations.
Union,
cast,
final,
+ overload,
)
import warnings
import numpy as np
-from pandas._config.config import option_context
-
from pandas._libs import (
Timestamp,
lib,
@@ -46,16 +45,12 @@ class providing the base-class of operations.
from pandas._typing import (
AnyArrayLike,
ArrayLike,
- Axis,
- AxisInt,
DtypeObj,
- FillnaOptions,
IndexLabel,
+ IntervalClosedType,
NDFrameT,
PositionalIndexer,
RandomState,
- Scalar,
- T,
npt,
)
from pandas.compat.numpy import function as nv
@@ -99,11 +94,9 @@ class providing the base-class of operations.
sample,
)
from pandas.core._numba import executor
-from pandas.core.apply import warn_alias_replacement
from pandas.core.arrays import (
ArrowExtensionArray,
BaseMaskedArray,
- Categorical,
ExtensionArray,
FloatingArray,
IntegerArray,
@@ -132,7 +125,6 @@ class providing the base-class of operations.
GroupByNthSelector,
)
from pandas.core.indexes.api import (
- CategoricalIndex,
Index,
MultiIndex,
RangeIndex,
@@ -147,8 +139,16 @@ class providing the base-class of operations.
)
if TYPE_CHECKING:
- from typing import Any
+ from pandas._libs.tslibs import BaseOffset
+ from pandas._typing import (
+ Any,
+ Concatenate,
+ P,
+ Self,
+ T,
+ )
+ from pandas.core.indexers.objects import BaseIndexer
from pandas.core.resample import Resampler
from pandas.core.window import (
ExpandingGroupby,
@@ -164,191 +164,6 @@ class providing the base-class of operations.
to each row or column of a DataFrame.
"""
-_apply_docs = {
- "template": """
- Apply function ``func`` group-wise and combine the results together.
-
- The function passed to ``apply`` must take a {input} as its first
- argument and return a DataFrame, Series or scalar. ``apply`` will
- then take care of combining the results back together into a single
- dataframe or series. ``apply`` is therefore a highly flexible
- grouping method.
-
- While ``apply`` is a very flexible method, its downside is that
- using it can be quite a bit slower than using more specific methods
- like ``agg`` or ``transform``. Pandas offers a wide range of method that will
- be much faster than using ``apply`` for their specific purposes, so try to
- use them before reaching for ``apply``.
-
- Parameters
- ----------
- func : callable
- A callable that takes a {input} as its first argument, and
- returns a dataframe, a series or a scalar. In addition the
- callable may take positional and keyword arguments.
- include_groups : bool, default True
- When True, will attempt to apply ``func`` to the groupings in
- the case that they are columns of the DataFrame. If this raises a
- TypeError, the result will be computed with the groupings excluded.
- When False, the groupings will be excluded when applying ``func``.
-
- .. versionadded:: 2.2.0
-
- .. deprecated:: 2.2.0
-
- Setting include_groups to True is deprecated. Only the value
- False will be allowed in a future version of pandas.
-
- args, kwargs : tuple and dict
- Optional positional and keyword arguments to pass to ``func``.
-
- Returns
- -------
- Series or DataFrame
-
- See Also
- --------
- pipe : Apply function to the full GroupBy object instead of to each
- group.
- aggregate : Apply aggregate function to the GroupBy object.
- transform : Apply function column-by-column to the GroupBy object.
- Series.apply : Apply a function to a Series.
- DataFrame.apply : Apply a function to each row or column of a DataFrame.
-
- Notes
- -----
-
- .. versionchanged:: 1.3.0
-
- The resulting dtype will reflect the return value of the passed ``func``,
- see the examples below.
-
- Functions that mutate the passed object can produce unexpected
- behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
- for more details.
-
- Examples
- --------
- {examples}
- """,
- "dataframe_examples": """
- >>> df = pd.DataFrame({'A': 'a a b'.split(),
- ... 'B': [1, 2, 3],
- ... 'C': [4, 6, 5]})
- >>> g1 = df.groupby('A', group_keys=False)
- >>> g2 = df.groupby('A', group_keys=True)
-
- Notice that ``g1`` and ``g2`` have two groups, ``a`` and ``b``, and only
- differ in their ``group_keys`` argument. Calling `apply` in various ways,
- we can get different grouping results:
-
- Example 1: below the function passed to `apply` takes a DataFrame as
- its argument and returns a DataFrame. `apply` combines the result for
- each group together into a new DataFrame:
-
- >>> g1[['B', 'C']].apply(lambda x: x / x.sum())
- B C
- 0 0.333333 0.4
- 1 0.666667 0.6
- 2 1.000000 1.0
-
- In the above, the groups are not part of the index. We can have them included
- by using ``g2`` where ``group_keys=True``:
-
- >>> g2[['B', 'C']].apply(lambda x: x / x.sum())
- B C
- A
- a 0 0.333333 0.4
- 1 0.666667 0.6
- b 2 1.000000 1.0
-
- Example 2: The function passed to `apply` takes a DataFrame as
- its argument and returns a Series. `apply` combines the result for
- each group together into a new DataFrame.
-
- .. versionchanged:: 1.3.0
-
- The resulting dtype will reflect the return value of the passed ``func``.
-
- >>> g1[['B', 'C']].apply(lambda x: x.astype(float).max() - x.min())
- B C
- A
- a 1.0 2.0
- b 0.0 0.0
-
- >>> g2[['B', 'C']].apply(lambda x: x.astype(float).max() - x.min())
- B C
- A
- a 1.0 2.0
- b 0.0 0.0
-
- The ``group_keys`` argument has no effect here because the result is not
- like-indexed (i.e. :ref:`a transform `) when compared
- to the input.
-
- Example 3: The function passed to `apply` takes a DataFrame as
- its argument and returns a scalar. `apply` combines the result for
- each group together into a Series, including setting the index as
- appropriate:
-
- >>> g1.apply(lambda x: x.C.max() - x.B.min(), include_groups=False)
- A
- a 5
- b 2
- dtype: int64""",
- "series_examples": """
- >>> s = pd.Series([0, 1, 2], index='a a b'.split())
- >>> g1 = s.groupby(s.index, group_keys=False)
- >>> g2 = s.groupby(s.index, group_keys=True)
-
- From ``s`` above we can see that ``g`` has two groups, ``a`` and ``b``.
- Notice that ``g1`` have ``g2`` have two groups, ``a`` and ``b``, and only
- differ in their ``group_keys`` argument. Calling `apply` in various ways,
- we can get different grouping results:
-
- Example 1: The function passed to `apply` takes a Series as
- its argument and returns a Series. `apply` combines the result for
- each group together into a new Series.
-
- .. versionchanged:: 1.3.0
-
- The resulting dtype will reflect the return value of the passed ``func``.
-
- >>> g1.apply(lambda x: x * 2 if x.name == 'a' else x / 2)
- a 0.0
- a 2.0
- b 1.0
- dtype: float64
-
- In the above, the groups are not part of the index. We can have them included
- by using ``g2`` where ``group_keys=True``:
-
- >>> g2.apply(lambda x: x * 2 if x.name == 'a' else x / 2)
- a a 0.0
- a 2.0
- b b 1.0
- dtype: float64
-
- Example 2: The function passed to `apply` takes a Series as
- its argument and returns a scalar. `apply` combines the result for
- each group together into a Series, including setting the index as
- appropriate:
-
- >>> g1.apply(lambda x: x.max() - x.min())
- a 1
- b 0
- dtype: int64
-
- The ``group_keys`` argument has no effect here because the result is not
- like-indexed (i.e. :ref:`a transform `) when compared
- to the input.
-
- >>> g2.apply(lambda x: x.max() - x.min())
- a 1
- b 0
- dtype: int64""",
-}
-
_groupby_agg_method_template = """
Compute {fname} of group values.
@@ -444,14 +259,15 @@ class providing the base-class of operations.
a `(callable, data_keyword)` tuple where `data_keyword` is a
string indicating the keyword of `callable` that expects the
%(klass)s object.
-args : iterable, optional
+*args : iterable, optional
Positional arguments passed into `func`.
-kwargs : dict, optional
+**kwargs : dict, optional
A dictionary of keyword arguments passed into `func`.
Returns
-------
-the return type of `func`.
+%(klass)s
+ The original object with the function `func` applied.
See Also
--------
@@ -478,7 +294,7 @@ class providing the base-class of operations.
Parameters
----------
-f : function, str
+func : function, str
Function to apply to each group. See the Notes section below for requirements.
Accepted inputs are:
@@ -522,8 +338,7 @@ class providing the base-class of operations.
--------
%(klass)s.groupby.apply : Apply function ``func`` group-wise and combine
the results together.
-%(klass)s.groupby.aggregate : Aggregate using one or more
- operations over the specified axis.
+%(klass)s.groupby.aggregate : Aggregate using one or more operations.
%(klass)s.transform : Call ``func`` on self producing a %(klass)s with the
same axis shape as self.
@@ -565,7 +380,7 @@ class providing the base-class of operations.
%(example)s"""
_agg_template_series = """
-Aggregate using one or more operations over the specified axis.
+Aggregate using one or more operations.
Parameters
----------
@@ -627,8 +442,7 @@ class providing the base-class of operations.
and combine the results together.
{klass}.groupby.transform : Transforms the Series on each group
based on the given function.
-{klass}.aggregate : Aggregate using one or more
- operations over the specified axis.
+{klass}.aggregate : Aggregate using one or more operations.
Notes
-----
@@ -647,7 +461,7 @@ class providing the base-class of operations.
{examples}"""
_agg_template_frame = """
-Aggregate using one or more operations over the specified axis.
+Aggregate using one or more operations.
Parameters
----------
@@ -660,7 +474,7 @@ class providing the base-class of operations.
- function
- string function name
- list of functions and/or function names, e.g. ``[np.sum, 'mean']``
- - dict of axis labels -> functions, function names or list of such.
+ - dict of index labels -> functions, function names or list of such.
- None, in which case ``**kwargs`` are used with Named Aggregation. Here the
output has one column for each element in ``**kwargs``. The name of the
column is keyword, whereas the value determines the aggregation used to compute
@@ -706,8 +520,7 @@ class providing the base-class of operations.
and combine the results together.
{klass}.groupby.transform : Transforms the Series on each group
based on the given function.
-{klass}.aggregate : Aggregate using one or more
- operations over the specified axis.
+{klass}.aggregate : Aggregate using one or more operations.
Notes
-----
@@ -764,7 +577,6 @@ def f(self):
class BaseGroupBy(PandasObject, SelectionMixin[NDFrameT], GroupByIndexingMixin):
_hidden_attrs = PandasObject._hidden_attrs | {
"as_index",
- "axis",
"dropna",
"exclusions",
"grouper",
@@ -776,7 +588,6 @@ class BaseGroupBy(PandasObject, SelectionMixin[NDFrameT], GroupByIndexingMixin):
"sort",
}
- axis: AxisInt
_grouper: ops.BaseGrouper
keys: _KeysArgType | None = None
level: IndexLabel | None = None
@@ -784,7 +595,7 @@ class BaseGroupBy(PandasObject, SelectionMixin[NDFrameT], GroupByIndexingMixin):
@final
def __len__(self) -> int:
- return len(self.groups)
+ return self._grouper.ngroups
@final
def __repr__(self) -> str:
@@ -793,18 +604,7 @@ def __repr__(self) -> str:
@final
@property
- def grouper(self) -> ops.BaseGrouper:
- warnings.warn(
- f"{type(self).__name__}.grouper is deprecated and will be removed in a "
- "future version of pandas.",
- category=FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self._grouper
-
- @final
- @property
- def groups(self) -> dict[Hashable, np.ndarray]:
+ def groups(self) -> dict[Hashable, Index]:
"""
Dict {group name -> group labels}.
@@ -813,7 +613,7 @@ def groups(self) -> dict[Hashable, np.ndarray]:
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b']
+ >>> lst = ["a", "a", "b"]
>>> ser = pd.Series([1, 2, 3], index=lst)
>>> ser
a 1
@@ -837,15 +637,19 @@ def groups(self) -> dict[Hashable, np.ndarray]:
For Resampler:
- >>> ser = pd.Series([1, 2, 3, 4], index=pd.DatetimeIndex(
- ... ['2023-01-01', '2023-01-15', '2023-02-01', '2023-02-15']))
+ >>> ser = pd.Series(
+ ... [1, 2, 3, 4],
+ ... index=pd.DatetimeIndex(
+ ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
+ ... ),
+ ... )
>>> ser
2023-01-01 1
2023-01-15 2
2023-02-01 3
2023-02-15 4
dtype: int64
- >>> ser.resample('MS').groups
+ >>> ser.resample("MS").groups
{Timestamp('2023-01-01 00:00:00'): 2, Timestamp('2023-02-01 00:00:00'): 4}
"""
return self._grouper.groups
@@ -866,7 +670,7 @@ def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b']
+ >>> lst = ["a", "a", "b"]
>>> ser = pd.Series([1, 2, 3], index=lst)
>>> ser
a 1
@@ -879,8 +683,9 @@ def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
For DataFrameGroupBy:
>>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["owl", "toucan", "eagle"])
+ >>> df = pd.DataFrame(
+ ... data, columns=["a", "b", "c"], index=["owl", "toucan", "eagle"]
+ ... )
>>> df
a b c
owl 1 2 3
@@ -891,15 +696,19 @@ def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
For Resampler:
- >>> ser = pd.Series([1, 2, 3, 4], index=pd.DatetimeIndex(
- ... ['2023-01-01', '2023-01-15', '2023-02-01', '2023-02-15']))
+ >>> ser = pd.Series(
+ ... [1, 2, 3, 4],
+ ... index=pd.DatetimeIndex(
+ ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
+ ... ),
+ ... )
>>> ser
2023-01-01 1
2023-01-15 2
2023-02-01 3
2023-02-15 4
dtype: int64
- >>> ser.resample('MS').indices
+ >>> ser.resample("MS").indices
defaultdict(, {Timestamp('2023-01-01 00:00:00'): [0, 1],
Timestamp('2023-02-01 00:00:00'): [2, 3]})
"""
@@ -988,6 +797,22 @@ def _selected_obj(self):
def _dir_additions(self) -> set[str]:
return self.obj._dir_additions()
+ @overload
+ def pipe(
+ self,
+ func: Callable[Concatenate[Self, P], T],
+ *args: P.args,
+ **kwargs: P.kwargs,
+ ) -> T: ...
+
+ @overload
+ def pipe(
+ self,
+ func: tuple[Callable[..., T], str],
+ *args: Any,
+ **kwargs: Any,
+ ) -> T: ...
+
@Substitution(
klass="GroupBy",
examples=dedent(
@@ -1013,14 +838,14 @@ def _dir_additions(self) -> set[str]:
@Appender(_pipe_template)
def pipe(
self,
- func: Callable[..., T] | tuple[Callable[..., T], str],
- *args,
- **kwargs,
+ func: Callable[Concatenate[Self, P], T] | tuple[Callable[..., T], str],
+ *args: Any,
+ **kwargs: Any,
) -> T:
return com.pipe(self, func, *args, **kwargs)
@final
- def get_group(self, name, obj=None) -> DataFrame | Series:
+ def get_group(self, name) -> DataFrame | Series:
"""
Construct DataFrame from group with provided name.
@@ -1028,26 +853,17 @@ def get_group(self, name, obj=None) -> DataFrame | Series:
----------
name : object
The name of the group to get as a DataFrame.
- obj : DataFrame, default None
- The DataFrame to take the DataFrame out of. If
- it is None, the object groupby was called on will
- be used.
-
- .. deprecated:: 2.1.0
- The obj is deprecated and will be removed in a future version.
- Do ``df.iloc[gb.indices.get(name)]``
- instead of ``gb.get_group(name, obj=df)``.
Returns
-------
- same type as obj
+ DataFrame or Series
Examples
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b']
+ >>> lst = ["a", "a", "b"]
>>> ser = pd.Series([1, 2, 3], index=lst)
>>> ser
a 1
@@ -1062,8 +878,9 @@ def get_group(self, name, obj=None) -> DataFrame | Series:
For DataFrameGroupBy:
>>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["owl", "toucan", "eagle"])
+ >>> df = pd.DataFrame(
+ ... data, columns=["a", "b", "c"], index=["owl", "toucan", "eagle"]
+ ... )
>>> df
a b c
owl 1 2 3
@@ -1076,15 +893,19 @@ def get_group(self, name, obj=None) -> DataFrame | Series:
For Resampler:
- >>> ser = pd.Series([1, 2, 3, 4], index=pd.DatetimeIndex(
- ... ['2023-01-01', '2023-01-15', '2023-02-01', '2023-02-15']))
+ >>> ser = pd.Series(
+ ... [1, 2, 3, 4],
+ ... index=pd.DatetimeIndex(
+ ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
+ ... ),
+ ... )
>>> ser
2023-01-01 1
2023-01-15 2
2023-02-01 3
2023-02-15 4
dtype: int64
- >>> ser.resample('MS').get_group('2023-01-01')
+ >>> ser.resample("MS").get_group("2023-01-01")
2023-01-01 1
2023-01-15 2
dtype: int64
@@ -1097,34 +918,14 @@ def get_group(self, name, obj=None) -> DataFrame | Series:
):
# GH#25971
if isinstance(name, tuple) and len(name) == 1:
- # Allow users to pass tuples of length 1 to silence warning
name = name[0]
- elif not isinstance(name, tuple):
- warnings.warn(
- "When grouping with a length-1 list-like, "
- "you will need to pass a length-1 tuple to get_group in a future "
- "version of pandas. Pass `(name,)` instead of `name` to silence "
- "this warning.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
+ else:
+ raise KeyError(name)
inds = self._get_index(name)
if not len(inds):
raise KeyError(name)
-
- if obj is None:
- indexer = inds if self.axis == 0 else (slice(None), inds)
- return self._selected_obj.iloc[indexer]
- else:
- warnings.warn(
- "obj is deprecated and will be removed in a future version. "
- "Do ``df.iloc[gb.indices.get(name)]`` "
- "instead of ``gb.get_group(name, obj=df)``.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return obj._take_with_is_copy(inds, axis=self.axis)
+ return self._selected_obj.iloc[inds]
@final
def __iter__(self) -> Iterator[tuple[Hashable, NDFrameT]]:
@@ -1141,7 +942,7 @@ def __iter__(self) -> Iterator[tuple[Hashable, NDFrameT]]:
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b']
+ >>> lst = ["a", "a", "b"]
>>> ser = pd.Series([1, 2, 3], index=lst)
>>> ser
a 1
@@ -1149,7 +950,7 @@ def __iter__(self) -> Iterator[tuple[Hashable, NDFrameT]]:
b 3
dtype: int64
>>> for x, y in ser.groupby(level=0):
- ... print(f'{x}\\n{y}\\n')
+ ... print(f"{x}\\n{y}\\n")
a
a 1
a 2
@@ -1168,7 +969,7 @@ def __iter__(self) -> Iterator[tuple[Hashable, NDFrameT]]:
1 1 5 6
2 7 8 9
>>> for x, y in df.groupby(by=["a"]):
- ... print(f'{x}\\n{y}\\n')
+ ... print(f"{x}\\n{y}\\n")
(1,)
a b c
0 1 2 3
@@ -1179,16 +980,20 @@ def __iter__(self) -> Iterator[tuple[Hashable, NDFrameT]]:
For Resampler:
- >>> ser = pd.Series([1, 2, 3, 4], index=pd.DatetimeIndex(
- ... ['2023-01-01', '2023-01-15', '2023-02-01', '2023-02-15']))
+ >>> ser = pd.Series(
+ ... [1, 2, 3, 4],
+ ... index=pd.DatetimeIndex(
+ ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
+ ... ),
+ ... )
>>> ser
2023-01-01 1
2023-01-15 2
2023-02-01 3
2023-02-15 4
dtype: int64
- >>> for x, y in ser.resample('MS'):
- ... print(f'{x}\\n{y}\\n')
+ >>> for x, y in ser.resample("MS"):
+ ... print(f"{x}\\n{y}\\n")
2023-01-01 00:00:00
2023-01-01 1
2023-01-15 2
@@ -1200,19 +1005,12 @@ def __iter__(self) -> Iterator[tuple[Hashable, NDFrameT]]:
"""
keys = self.keys
level = self.level
- result = self._grouper.get_iterator(self._selected_obj, axis=self.axis)
- # error: Argument 1 to "len" has incompatible type "Hashable"; expected "Sized"
- if is_list_like(level) and len(level) == 1: # type: ignore[arg-type]
- # GH 51583
- warnings.warn(
- "Creating a Groupby object with a length-1 list-like "
- "level parameter will yield indexes as tuples in a future version. "
- "To keep indexes as scalars, create Groupby objects with "
- "a scalar level parameter instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- if isinstance(keys, list) and len(keys) == 1:
+ result = self._grouper.get_iterator(self._selected_obj)
+ # mypy: Argument 1 to "len" has incompatible type "Hashable"; expected "Sized"
+ if (
+ (is_list_like(level) and len(level) == 1) # type: ignore[arg-type]
+ or (isinstance(keys, list) and len(keys) == 1)
+ ):
# GH#42795 - when keys is a list, return tuples even when length is 1
result = (((key,), group) for key, group in result)
return result
@@ -1237,7 +1035,6 @@ class GroupBy(BaseGroupBy[NDFrameT]):
Parameters
----------
obj : pandas object
- axis : int, default 0
level : int, default None
Level of MultiIndex
groupings : list of Grouping objects
@@ -1266,7 +1063,7 @@ class GroupBy(BaseGroupBy[NDFrameT]):
::
- grouped = obj.groupby(keys, axis=axis)
+ grouped = obj.groupby(keys)
for key, group in grouped:
# do something with the data
@@ -1298,7 +1095,6 @@ def __init__(
self,
obj: NDFrameT,
keys: _KeysArgType | None = None,
- axis: Axis = 0,
level: IndexLabel | None = None,
grouper: ops.BaseGrouper | None = None,
exclusions: frozenset[Hashable] | None = None,
@@ -1306,7 +1102,7 @@ def __init__(
as_index: bool = True,
sort: bool = True,
group_keys: bool = True,
- observed: bool | lib.NoDefault = lib.no_default,
+ observed: bool = False,
dropna: bool = True,
) -> None:
self._selection = selection
@@ -1314,11 +1110,6 @@ def __init__(
assert isinstance(obj, NDFrame), type(obj)
self.level = level
-
- if not as_index:
- if axis != 0:
- raise ValueError("as_index=False only valid for axis=0")
-
self.as_index = as_index
self.keys = keys
self.sort = sort
@@ -1329,28 +1120,14 @@ def __init__(
grouper, exclusions, obj = get_grouper(
obj,
keys,
- axis=axis,
level=level,
sort=sort,
- observed=False if observed is lib.no_default else observed,
+ observed=observed,
dropna=self.dropna,
)
- if observed is lib.no_default:
- if any(ping._passed_categorical for ping in grouper.groupings):
- warnings.warn(
- "The default of observed=False is deprecated and will be changed "
- "to True in a future version of pandas. Pass observed=False to "
- "retain current behavior or observed=True to adopt the future "
- "default and silence this warning.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- observed = False
self.observed = observed
-
self.obj = obj
- self.axis = obj._get_axis_number(axis)
self._grouper = grouper
self.exclusions = frozenset(exclusions) if exclusions else frozenset()
@@ -1364,50 +1141,10 @@ def __getattr__(self, attr: str):
f"'{type(self).__name__}' object has no attribute '{attr}'"
)
- @final
- def _deprecate_axis(self, axis: int, name: str) -> None:
- if axis == 1:
- warnings.warn(
- f"{type(self).__name__}.{name} with axis=1 is deprecated and "
- "will be removed in a future version. Operate on the un-grouped "
- "DataFrame instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- warnings.warn(
- f"The 'axis' keyword in {type(self).__name__}.{name} is deprecated "
- "and will be removed in a future version. "
- "Call without passing 'axis' instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
@final
def _op_via_apply(self, name: str, *args, **kwargs):
"""Compute the result of an operation by using GroupBy's apply."""
f = getattr(type(self._obj_with_exclusions), name)
- sig = inspect.signature(f)
-
- if "axis" in kwargs and kwargs["axis"] is not lib.no_default:
- axis = self.obj._get_axis_number(kwargs["axis"])
- self._deprecate_axis(axis, name)
- elif "axis" in kwargs:
- # exclude skew here because that was already defaulting to lib.no_default
- # before this deprecation was instituted
- if name == "skew":
- pass
- elif name == "fillna":
- # maintain the behavior from before the deprecation
- kwargs["axis"] = None
- else:
- kwargs["axis"] = 0
-
- # a little trickery for aggregation functions that need an axis
- # argument
- if "axis" in sig.parameters:
- if kwargs.get("axis", None) is None or kwargs.get("axis") is lib.no_default:
- kwargs["axis"] = self.axis
def curried(x):
return f(x, *args, **kwargs)
@@ -1456,7 +1193,7 @@ def _concat_objects(
result = concat(
values,
- axis=self.axis,
+ axis=0,
keys=group_keys,
levels=group_levels,
names=group_names,
@@ -1465,15 +1202,15 @@ def _concat_objects(
else:
# GH5610, returns a MI, with the first level being a
# range index
- keys = list(range(len(values)))
- result = concat(values, axis=self.axis, keys=keys)
+ keys = RangeIndex(len(values))
+ result = concat(values, axis=0, keys=keys)
elif not not_indexed_same:
- result = concat(values, axis=self.axis)
+ result = concat(values, axis=0)
- ax = self._selected_obj._get_axis(self.axis)
+ ax = self._selected_obj.index
if self.dropna:
- labels = self._grouper.group_info[0]
+ labels = self._grouper.ids
mask = labels != -1
ax = ax[mask]
@@ -1483,16 +1220,16 @@ def _concat_objects(
# so we resort to this
# GH 14776, 30667
# TODO: can we reuse e.g. _reindex_non_unique?
- if ax.has_duplicates and not result.axes[self.axis].equals(ax):
+ if ax.has_duplicates and not result.axes[0].equals(ax):
# e.g. test_category_order_transformer
target = algorithms.unique1d(ax._values)
indexer, _ = result.index.get_indexer_non_unique(target)
- result = result.take(indexer, axis=self.axis)
+ result = result.take(indexer, axis=0)
else:
- result = result.reindex(ax, axis=self.axis, copy=False)
+ result = result.reindex(ax, axis=0)
else:
- result = concat(values, axis=self.axis)
+ result = concat(values, axis=0)
if self.obj.ndim == 1:
name = self.obj.name
@@ -1513,66 +1250,64 @@ def _set_result_index_ordered(
# set the result index on the passed values object and
# return the new object, xref 8046
- obj_axis = self.obj._get_axis(self.axis)
+ index = self.obj.index
if self._grouper.is_monotonic and not self._grouper.has_dropped_na:
# shortcut if we have an already ordered grouper
- result = result.set_axis(obj_axis, axis=self.axis, copy=False)
+ result = result.set_axis(index, axis=0)
return result
# row order is scrambled => sort the rows by position in original index
- original_positions = Index(self._grouper.result_ilocs())
- result = result.set_axis(original_positions, axis=self.axis, copy=False)
- result = result.sort_index(axis=self.axis)
+ original_positions = Index(self._grouper.result_ilocs)
+ result = result.set_axis(original_positions, axis=0)
+ result = result.sort_index(axis=0)
if self._grouper.has_dropped_na:
# Add back in any missing rows due to dropna - index here is integral
# with values referring to the row of the input so can use RangeIndex
- result = result.reindex(RangeIndex(len(obj_axis)), axis=self.axis)
- result = result.set_axis(obj_axis, axis=self.axis, copy=False)
+ result = result.reindex(RangeIndex(len(index)), axis=0)
+ result = result.set_axis(index, axis=0)
return result
@final
- def _insert_inaxis_grouper(self, result: Series | DataFrame) -> DataFrame:
+ def _insert_inaxis_grouper(
+ self, result: Series | DataFrame, qs: npt.NDArray[np.float64] | None = None
+ ) -> DataFrame:
if isinstance(result, Series):
result = result.to_frame()
+ n_groupings = len(self._grouper.groupings)
+
+ if qs is not None:
+ result.insert(
+ 0, f"level_{n_groupings}", np.tile(qs, len(result) // len(qs))
+ )
+
# zip in reverse so we can always insert at loc 0
- columns = result.columns
- for name, lev, in_axis in zip(
- reversed(self._grouper.names),
- reversed(self._grouper.get_group_levels()),
- reversed([grp.in_axis for grp in self._grouper.groupings]),
+ for level, (name, lev, in_axis) in enumerate(
+ zip(
+ reversed(self._grouper.names),
+ reversed(self._grouper.get_group_levels()),
+ reversed([grp.in_axis for grp in self._grouper.groupings]),
+ )
):
+ if name is None:
+ # Behave the same as .reset_index() when a level is unnamed
+ name = (
+ "index"
+ if n_groupings == 1 and qs is None
+ else f"level_{n_groupings - level - 1}"
+ )
+
# GH #28549
# When using .apply(-), name will be in columns already
- if name not in columns:
- if in_axis:
+ if name not in result.columns:
+ # if in_axis:
+ if qs is None:
result.insert(0, name, lev)
else:
- msg = (
- "A grouping was used that is not in the columns of the "
- "DataFrame and so was excluded from the result. This grouping "
- "will be included in a future version of pandas. Add the "
- "grouping as a column of the DataFrame to silence this warning."
- )
- warnings.warn(
- message=msg,
- category=FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- return result
+ result.insert(0, name, Index(np.repeat(lev, len(qs))))
- @final
- def _maybe_transpose_result(self, result: NDFrameT) -> NDFrameT:
- if self.axis == 1:
- # Only relevant for DataFrameGroupBy, no-op for SeriesGroupBy
- result = result.T
- if result.index.equals(self.obj.index):
- # Retain e.g. DatetimeIndex/TimedeltaIndex freq
- # e.g. test_groupby_crash_on_nunique
- result.index = self.obj.index.copy()
return result
@final
@@ -1598,24 +1333,19 @@ def _wrap_aggregated_output(
if not self.as_index:
# `not self.as_index` is only relevant for DataFrameGroupBy,
# enforced in __init__
- result = self._insert_inaxis_grouper(result)
+ result = self._insert_inaxis_grouper(result, qs=qs)
result = result._consolidate()
- index = Index(range(self._grouper.ngroups))
+ result.index = RangeIndex(len(result))
else:
index = self._grouper.result_index
+ if qs is not None:
+ # We get here with len(qs) != 1 and not self.as_index
+ # in test_pass_args_kwargs
+ index = _insert_quantile_level(index, qs)
+ result.index = index
- if qs is not None:
- # We get here with len(qs) != 1 and not self.as_index
- # in test_pass_args_kwargs
- index = _insert_quantile_level(index, qs)
-
- result.index = index
-
- # error: Argument 1 to "_maybe_transpose_result" of "GroupBy" has
- # incompatible type "Union[Series, DataFrame]"; expected "NDFrameT"
- res = self._maybe_transpose_result(result) # type: ignore[arg-type]
- return self._reindex_output(res, qs=qs)
+ return result
def _wrap_applied_output(
self,
@@ -1631,11 +1361,11 @@ def _wrap_applied_output(
@final
def _numba_prep(self, data: DataFrame):
- ids, _, ngroups = self._grouper.group_info
- sorted_index = self._grouper._sort_idx
+ ids, ngroups = self._grouper.group_info
+ sorted_index = self._grouper.result_ilocs
sorted_ids = self._grouper._sorted_ids
- sorted_data = data.take(sorted_index, axis=self.axis).to_numpy()
+ sorted_data = data.take(sorted_index, axis=0).to_numpy()
# GH 46867
index_data = data.index
if isinstance(index_data, MultiIndex):
@@ -1671,8 +1401,6 @@ def _numba_agg_general(
raise NotImplementedError(
"as_index=False is not supported. Use .reset_index() instead."
)
- if self.axis == 1:
- raise NotImplementedError("axis=1 is not supported.")
data = self._obj_with_exclusions
df = data if data.ndim == 2 else data.to_frame()
@@ -1685,7 +1413,7 @@ def _numba_agg_general(
)
# Pass group ids to kernel directly if it can handle it
# (This is faster since it doesn't require a sort)
- ids, _, _ = self._grouper.group_info
+ ids = self._grouper.ids
ngroups = self._grouper.ngroups
res_mgr = df._mgr.apply(
@@ -1711,6 +1439,7 @@ def _transform_with_numba(self, func, *args, engine_kwargs=None, **kwargs):
data and indices into a Numba jitted function.
"""
data = self._obj_with_exclusions
+ index_sorting = self._grouper.result_ilocs
df = data if data.ndim == 2 else data.to_frame()
starts, ends, sorted_index, sorted_data = self._numba_prep(df)
@@ -1728,7 +1457,7 @@ def _transform_with_numba(self, func, *args, engine_kwargs=None, **kwargs):
)
# result values needs to be resorted to their original positions since we
# evaluated the data sorted by group
- result = result.take(np.argsort(sorted_index), axis=0)
+ result = result.take(np.argsort(index_sorting), axis=0)
index = data.index
if data.ndim == 1:
result_kwargs = {"name": data.name}
@@ -1777,18 +1506,146 @@ def _aggregate_with_numba(self, func, *args, engine_kwargs=None, **kwargs):
# -----------------------------------------------------------------
# apply/agg/transform
- @Appender(
- _apply_docs["template"].format(
- input="dataframe", examples=_apply_docs["dataframe_examples"]
- )
- )
def apply(self, func, *args, include_groups: bool = True, **kwargs) -> NDFrameT:
- orig_func = func
- func = com.is_builtin_func(func)
- if orig_func != func:
- alias = com._builtin_table_alias[orig_func]
- warn_alias_replacement(self, orig_func, alias)
+ """
+ Apply function ``func`` group-wise and combine the results together.
+
+ The function passed to ``apply`` must take a dataframe as its first
+ argument and return a DataFrame, Series or scalar. ``apply`` will
+ then take care of combining the results back together into a single
+ dataframe or series. ``apply`` is therefore a highly flexible
+ grouping method.
+
+ While ``apply`` is a very flexible method, its downside is that
+ using it can be quite a bit slower than using more specific methods
+ like ``agg`` or ``transform``. Pandas offers a wide range of method that will
+ be much faster than using ``apply`` for their specific purposes, so try to
+ use them before reaching for ``apply``.
+ Parameters
+ ----------
+ func : callable
+ A callable that takes a dataframe as its first argument, and
+ returns a dataframe, a series or a scalar. In addition the
+ callable may take positional and keyword arguments.
+
+ *args : tuple
+ Optional positional arguments to pass to ``func``.
+
+ include_groups : bool, default True
+ When True, will attempt to apply ``func`` to the groupings in
+ the case that they are columns of the DataFrame. If this raises a
+ TypeError, the result will be computed with the groupings excluded.
+ When False, the groupings will be excluded when applying ``func``.
+
+ .. versionadded:: 2.2.0
+
+ .. deprecated:: 2.2.0
+
+ Setting include_groups to True is deprecated. Only the value
+ False will be allowed in a future version of pandas.
+
+ **kwargs : dict
+ Optional keyword arguments to pass to ``func``.
+
+ Returns
+ -------
+ Series or DataFrame
+
+ See Also
+ --------
+ pipe : Apply function to the full GroupBy object instead of to each
+ group.
+ aggregate : Apply aggregate function to the GroupBy object.
+ transform : Apply function column-by-column to the GroupBy object.
+ Series.apply : Apply a function to a Series.
+ DataFrame.apply : Apply a function to each row or column of a DataFrame.
+
+ Notes
+ -----
+
+ .. versionchanged:: 1.3.0
+
+ The resulting dtype will reflect the return value of the passed ``func``,
+ see the examples below.
+
+ Functions that mutate the passed object can produce unexpected
+ behavior or errors and are not supported. See :ref:`gotchas.udf-mutation`
+ for more details.
+
+ Examples
+ --------
+ >>> df = pd.DataFrame({"A": "a a b".split(), "B": [1, 2, 3], "C": [4, 6, 5]})
+ >>> g1 = df.groupby("A", group_keys=False)
+ >>> g2 = df.groupby("A", group_keys=True)
+
+ Notice that ``g1`` and ``g2`` have two groups, ``a`` and ``b``, and only
+ differ in their ``group_keys`` argument. Calling `apply` in various ways,
+ we can get different grouping results:
+
+ Example 1: below the function passed to `apply` takes a DataFrame as
+ its argument and returns a DataFrame. `apply` combines the result for
+ each group together into a new DataFrame:
+
+ >>> g1[["B", "C"]].apply(lambda x: x / x.sum())
+ B C
+ 0 0.333333 0.4
+ 1 0.666667 0.6
+ 2 1.000000 1.0
+
+ In the above, the groups are not part of the index. We can have them included
+ by using ``g2`` where ``group_keys=True``:
+
+ >>> g2[["B", "C"]].apply(lambda x: x / x.sum())
+ B C
+ A
+ a 0 0.333333 0.4
+ 1 0.666667 0.6
+ b 2 1.000000 1.0
+
+ Example 2: The function passed to `apply` takes a DataFrame as
+ its argument and returns a Series. `apply` combines the result for
+ each group together into a new DataFrame.
+
+ .. versionchanged:: 1.3.0
+
+ The resulting dtype will reflect the return value of the passed ``func``.
+
+ >>> g1[["B", "C"]].apply(lambda x: x.astype(float).max() - x.min())
+ B C
+ A
+ a 1.0 2.0
+ b 0.0 0.0
+
+ >>> g2[["B", "C"]].apply(lambda x: x.astype(float).max() - x.min())
+ B C
+ A
+ a 1.0 2.0
+ b 0.0 0.0
+
+ The ``group_keys`` argument has no effect here because the result is not
+ like-indexed (i.e. :ref:`a transform `) when compared
+ to the input.
+
+ Example 3: The function passed to `apply` takes a DataFrame as
+ its argument and returns a scalar. `apply` combines the result for
+ each group together into a Series, including setting the index as
+ appropriate:
+
+ >>> g1.apply(lambda x: x.C.max() - x.B.min(), include_groups=False)
+ A
+ a 5
+ b 2
+ dtype: int64
+
+ Example 4: The function passed to ``apply`` returns ``None`` for one of the
+ group. This group is filtered from the result:
+
+ >>> g1.apply(lambda x: None if x.iloc[0, 0] == 3 else x, include_groups=False)
+ B C
+ 0 1 4
+ 1 2 6
+ """
if isinstance(func, str):
if hasattr(self, func):
res = getattr(self, func)
@@ -1818,32 +1675,28 @@ def f(g):
if not include_groups:
return self._python_apply_general(f, self._obj_with_exclusions)
- # ignore SettingWithCopy here in case the user mutates
- with option_context("mode.chained_assignment", None):
- try:
- result = self._python_apply_general(f, self._selected_obj)
- if (
- not isinstance(self.obj, Series)
- and self._selection is None
- and self._selected_obj.shape != self._obj_with_exclusions.shape
- ):
- warnings.warn(
- message=_apply_groupings_depr.format(
- type(self).__name__, "apply"
- ),
- category=FutureWarning,
- stacklevel=find_stack_level(),
- )
- except TypeError:
- # gh-20949
- # try again, with .apply acting as a filtering
- # operation, by excluding the grouping column
- # This would normally not be triggered
- # except if the udf is trying an operation that
- # fails on *some* columns, e.g. a numeric operation
- # on a string grouper column
+ try:
+ result = self._python_apply_general(f, self._selected_obj)
+ if (
+ not isinstance(self.obj, Series)
+ and self._selection is None
+ and self._selected_obj.shape != self._obj_with_exclusions.shape
+ ):
+ warnings.warn(
+ message=_apply_groupings_depr.format(type(self).__name__, "apply"),
+ category=DeprecationWarning,
+ stacklevel=find_stack_level(),
+ )
+ except TypeError:
+ # gh-20949
+ # try again, with .apply acting as a filtering
+ # operation, by excluding the grouping column
+ # This would normally not be triggered
+ # except if the udf is trying an operation that
+ # fails on *some* columns, e.g. a numeric operation
+ # on a string grouper column
- return self._python_apply_general(f, self._obj_with_exclusions)
+ return self._python_apply_general(f, self._obj_with_exclusions)
return result
@@ -1882,7 +1735,7 @@ def _python_apply_general(
Series or DataFrame
data after applying f
"""
- values, mutated = self._grouper.apply_groupwise(f, data, self.axis)
+ values, mutated = self._grouper.apply_groupwise(f, data)
if not_indexed_same is None:
not_indexed_same = mutated
@@ -2000,23 +1853,14 @@ def array_func(values: ArrayLike) -> ArrayLike:
if how in ["idxmin", "idxmax"]:
res = self._wrap_idxmax_idxmin(res)
out = self._wrap_aggregated_output(res)
- if self.axis == 1:
- out = out.infer_objects(copy=False)
return out
- def _cython_transform(
- self, how: str, numeric_only: bool = False, axis: AxisInt = 0, **kwargs
- ):
+ def _cython_transform(self, how: str, numeric_only: bool = False, **kwargs):
raise AbstractMethodError(self)
@final
def _transform(self, func, *args, engine=None, engine_kwargs=None, **kwargs):
# optimized transforms
- orig_func = func
- func = com.get_cython_func(func) or func
- if orig_func != func:
- warn_alias_replacement(self, orig_func, func)
-
if not isinstance(func, str):
return self._transform_general(func, engine, engine_kwargs, *args, **kwargs)
@@ -2059,8 +1903,8 @@ def _wrap_transform_fast_result(self, result: NDFrameT) -> NDFrameT:
obj = self._obj_with_exclusions
# for each col, reshape to size of original frame by take operation
- ids, _, _ = self._grouper.group_info
- result = result.reindex(self._grouper.result_index, axis=self.axis, copy=False)
+ ids = self._grouper.ids
+ result = result.reindex(self._grouper.result_index, axis=0)
if self.obj.ndim == 1:
# i.e. SeriesGroupBy
@@ -2068,15 +1912,12 @@ def _wrap_transform_fast_result(self, result: NDFrameT) -> NDFrameT:
output = obj._constructor(out, index=obj.index, name=obj.name)
else:
# `.size()` gives Series output on DataFrame input, need axis 0
- axis = 0 if result.ndim == 1 else self.axis
# GH#46209
# Don't convert indices: negative indices need to give rise
# to null values in the result
- new_ax = result.axes[axis].take(ids)
- output = result._reindex_with_indexers(
- {axis: (new_ax, ids)}, allow_dups=True, copy=False
- )
- output = output.set_axis(obj._get_axis(self.axis), axis=axis)
+ new_ax = result.index.take(ids)
+ output = result._reindex_with_indexers({0: (new_ax, ids)}, allow_dups=True)
+ output = output.set_axis(obj.index, axis=0)
return output
# -----------------------------------------------------------------
@@ -2089,7 +1930,7 @@ def _apply_filter(self, indices, dropna):
else:
indices = np.sort(np.concatenate(indices))
if dropna:
- filtered = self._selected_obj.take(indices, axis=self.axis)
+ filtered = self._selected_obj.take(indices, axis=0)
else:
mask = np.empty(len(self._selected_obj.index), dtype=bool)
mask.fill(False)
@@ -2112,7 +1953,7 @@ def _cumcount_array(self, ascending: bool = True) -> np.ndarray:
this is currently implementing sort=False
(though the default is sort=True) for groupby in general
"""
- ids, _, ngroups = self._grouper.group_info
+ ids, ngroups = self._grouper.group_info
sorter = get_group_index_sorter(ids, ngroups)
ids, count = ids[sorter], len(ids)
@@ -2170,7 +2011,7 @@ def any(self, skipna: bool = True) -> NDFrameT:
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b']
+ >>> lst = ["a", "a", "b"]
>>> ser = pd.Series([1, 2, 0], index=lst)
>>> ser
a 1
@@ -2185,8 +2026,9 @@ def any(self, skipna: bool = True) -> NDFrameT:
For DataFrameGroupBy:
>>> data = [[1, 0, 3], [1, 0, 6], [7, 1, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["ostrich", "penguin", "parrot"])
+ >>> df = pd.DataFrame(
+ ... data, columns=["a", "b", "c"], index=["ostrich", "penguin", "parrot"]
+ ... )
>>> df
a b c
ostrich 1 0 3
@@ -2227,7 +2069,7 @@ def all(self, skipna: bool = True) -> NDFrameT:
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b']
+ >>> lst = ["a", "a", "b"]
>>> ser = pd.Series([1, 2, 0], index=lst)
>>> ser
a 1
@@ -2242,8 +2084,9 @@ def all(self, skipna: bool = True) -> NDFrameT:
For DataFrameGroupBy:
>>> data = [[1, 0, 3], [1, 5, 6], [7, 8, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["ostrich", "penguin", "parrot"])
+ >>> df = pd.DataFrame(
+ ... data, columns=["a", "b", "c"], index=["ostrich", "penguin", "parrot"]
+ ... )
>>> df
a b c
ostrich 1 0 3
@@ -2277,7 +2120,7 @@ def count(self) -> NDFrameT:
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b']
+ >>> lst = ["a", "a", "b"]
>>> ser = pd.Series([1, 2, np.nan], index=lst)
>>> ser
a 1.0
@@ -2292,8 +2135,9 @@ def count(self) -> NDFrameT:
For DataFrameGroupBy:
>>> data = [[1, np.nan, 3], [1, np.nan, 6], [7, 8, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["cow", "horse", "bull"])
+ >>> df = pd.DataFrame(
+ ... data, columns=["a", "b", "c"], index=["cow", "horse", "bull"]
+ ... )
>>> df
a b c
cow 1 NaN 3
@@ -2307,21 +2151,25 @@ def count(self) -> NDFrameT:
For Resampler:
- >>> ser = pd.Series([1, 2, 3, 4], index=pd.DatetimeIndex(
- ... ['2023-01-01', '2023-01-15', '2023-02-01', '2023-02-15']))
+ >>> ser = pd.Series(
+ ... [1, 2, 3, 4],
+ ... index=pd.DatetimeIndex(
+ ... ["2023-01-01", "2023-01-15", "2023-02-01", "2023-02-15"]
+ ... ),
+ ... )
>>> ser
2023-01-01 1
2023-01-15 2
2023-02-01 3
2023-02-15 4
dtype: int64
- >>> ser.resample('MS').count()
+ >>> ser.resample("MS").count()
2023-01-01 2
2023-02-01 2
Freq: MS, dtype: int64
"""
data = self._get_data_to_aggregate()
- ids, _, ngroups = self._grouper.group_info
+ ids, ngroups = self._grouper.group_info
mask = ids != -1
is_series = data.ndim == 1
@@ -2352,15 +2200,9 @@ def hfunc(bvalues: ArrayLike) -> ArrayLike:
new_mgr = data.grouped_reduce(hfunc)
new_obj = self._wrap_agged_manager(new_mgr)
+ result = self._wrap_aggregated_output(new_obj)
- # If we are grouping on categoricals we want unobserved categories to
- # return zero, rather than the default of NaN which the reindexing in
- # _wrap_aggregated_output() returns. GH 35028
- # e.g. test_dataframe_groupby_on_2_categoricals_when_observed_is_false
- with com.temp_setattr(self, "observed", True):
- result = self._wrap_aggregated_output(new_obj)
-
- return self._reindex_output(result, fill_value=0)
+ return result
@final
@Substitution(name="groupby")
@@ -2406,14 +2248,15 @@ def mean(
%(see_also)s
Examples
--------
- >>> df = pd.DataFrame({'A': [1, 1, 2, 1, 2],
- ... 'B': [np.nan, 2, 3, 4, 5],
- ... 'C': [1, 2, 1, 1, 2]}, columns=['A', 'B', 'C'])
+ >>> df = pd.DataFrame(
+ ... {"A": [1, 1, 2, 1, 2], "B": [np.nan, 2, 3, 4, 5], "C": [1, 2, 1, 1, 2]},
+ ... columns=["A", "B", "C"],
+ ... )
Groupby one column and return the mean of the remaining columns in
each group.
- >>> df.groupby('A').mean()
+ >>> df.groupby("A").mean()
B C
A
1 3.0 1.333333
@@ -2421,7 +2264,7 @@ def mean(
Groupby two columns and return the mean of the remaining column.
- >>> df.groupby(['A', 'B']).mean()
+ >>> df.groupby(["A", "B"]).mean()
C
A B
1 2.0 2.0
@@ -2432,7 +2275,7 @@ def mean(
Groupby one column and return the mean of only particular column in
the group.
- >>> df.groupby('A')['B'].mean()
+ >>> df.groupby("A")["B"].mean()
A
1 3.0
2 4.0
@@ -2481,7 +2324,7 @@ def median(self, numeric_only: bool = False) -> NDFrameT:
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'a', 'b', 'b', 'b']
+ >>> lst = ["a", "a", "a", "b", "b", "b"]
>>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
>>> ser
a 7
@@ -2498,9 +2341,10 @@ def median(self, numeric_only: bool = False) -> NDFrameT:
For DataFrameGroupBy:
- >>> data = {'a': [1, 3, 5, 7, 7, 8, 3], 'b': [1, 4, 8, 4, 4, 2, 1]}
- >>> df = pd.DataFrame(data, index=['dog', 'dog', 'dog',
- ... 'mouse', 'mouse', 'mouse', 'mouse'])
+ >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
+ >>> df = pd.DataFrame(
+ ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
+ ... )
>>> df
a b
dog 1 1
@@ -2517,14 +2361,20 @@ def median(self, numeric_only: bool = False) -> NDFrameT:
For Resampler:
- >>> ser = pd.Series([1, 2, 3, 3, 4, 5],
- ... index=pd.DatetimeIndex(['2023-01-01',
- ... '2023-01-10',
- ... '2023-01-15',
- ... '2023-02-01',
- ... '2023-02-10',
- ... '2023-02-15']))
- >>> ser.resample('MS').median()
+ >>> ser = pd.Series(
+ ... [1, 2, 3, 3, 4, 5],
+ ... index=pd.DatetimeIndex(
+ ... [
+ ... "2023-01-01",
+ ... "2023-01-10",
+ ... "2023-01-15",
+ ... "2023-02-01",
+ ... "2023-02-10",
+ ... "2023-02-15",
+ ... ]
+ ... ),
+ ... )
+ >>> ser.resample("MS").median()
2023-01-01 2.0
2023-02-01 4.0
Freq: MS, dtype: float64
@@ -2591,7 +2441,7 @@ def std(
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'a', 'b', 'b', 'b']
+ >>> lst = ["a", "a", "a", "b", "b", "b"]
>>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
>>> ser
a 7
@@ -2608,9 +2458,10 @@ def std(
For DataFrameGroupBy:
- >>> data = {'a': [1, 3, 5, 7, 7, 8, 3], 'b': [1, 4, 8, 4, 4, 2, 1]}
- >>> df = pd.DataFrame(data, index=['dog', 'dog', 'dog',
- ... 'mouse', 'mouse', 'mouse', 'mouse'])
+ >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
+ >>> df = pd.DataFrame(
+ ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
+ ... )
>>> df
a b
dog 1 1
@@ -2700,7 +2551,7 @@ def var(
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'a', 'b', 'b', 'b']
+ >>> lst = ["a", "a", "a", "b", "b", "b"]
>>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
>>> ser
a 7
@@ -2717,9 +2568,10 @@ def var(
For DataFrameGroupBy:
- >>> data = {'a': [1, 3, 5, 7, 7, 8, 3], 'b': [1, 4, 8, 4, 4, 2, 1]}
- >>> df = pd.DataFrame(data, index=['dog', 'dog', 'dog',
- ... 'mouse', 'mouse', 'mouse', 'mouse'])
+ >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
+ >>> df = pd.DataFrame(
+ ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
+ ... )
>>> df
a b
dog 1 1
@@ -2767,10 +2619,6 @@ def _value_counts(
SeriesGroupBy additionally supports a bins argument. See the docstring of
DataFrameGroupBy.value_counts for a description of arguments.
"""
- if self.axis == 1:
- raise NotImplementedError(
- "DataFrameGroupBy.value_counts only handles axis=0"
- )
name = "proportion" if normalize else "count"
df = self.obj
@@ -2813,7 +2661,6 @@ def _value_counts(
grouper, _, _ = get_grouper(
df,
key=key,
- axis=self.axis,
sort=self.sort,
observed=False,
dropna=dropna,
@@ -2830,19 +2677,6 @@ def _value_counts(
result_series = cast(Series, gb.size())
result_series.name = name
- # GH-46357 Include non-observed categories
- # of non-grouping columns regardless of `observed`
- if any(
- isinstance(grouping.grouping_vector, (Categorical, CategoricalIndex))
- and not grouping._observed
- for grouping in groupings
- ):
- levels_list = [ping._result_index for ping in groupings]
- multi_index = MultiIndex.from_product(
- levels_list, names=[ping.name for ping in groupings]
- )
- result_series = result_series.reindex(multi_index, fill_value=0)
-
if sort:
# Sort by the values
result_series = result_series.sort_values(
@@ -2853,7 +2687,7 @@ def _value_counts(
names = result_series.index.names
# GH#55951 - Temporarily replace names in case they are integers
result_series.index.names = range(len(names))
- index_level = list(range(len(self._grouper.groupings)))
+ index_level = range(len(self._grouper.groupings))
result_series = result_series.sort_index(
level=index_level, sort_remaining=False
)
@@ -2926,7 +2760,7 @@ def sem(self, ddof: int = 1, numeric_only: bool = False) -> NDFrameT:
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b', 'b']
+ >>> lst = ["a", "a", "b", "b"]
>>> ser = pd.Series([5, 10, 8, 14], index=lst)
>>> ser
a 5
@@ -2942,8 +2776,11 @@ def sem(self, ddof: int = 1, numeric_only: bool = False) -> NDFrameT:
For DataFrameGroupBy:
>>> data = [[1, 12, 11], [1, 15, 2], [2, 5, 8], [2, 6, 12]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["tuna", "salmon", "catfish", "goldfish"])
+ >>> df = pd.DataFrame(
+ ... data,
+ ... columns=["a", "b", "c"],
+ ... index=["tuna", "salmon", "catfish", "goldfish"],
+ ... )
>>> df
a b c
tuna 1 12 11
@@ -2958,14 +2795,20 @@ def sem(self, ddof: int = 1, numeric_only: bool = False) -> NDFrameT:
For Resampler:
- >>> ser = pd.Series([1, 3, 2, 4, 3, 8],
- ... index=pd.DatetimeIndex(['2023-01-01',
- ... '2023-01-10',
- ... '2023-01-15',
- ... '2023-02-01',
- ... '2023-02-10',
- ... '2023-02-15']))
- >>> ser.resample('MS').sem()
+ >>> ser = pd.Series(
+ ... [1, 3, 2, 4, 3, 8],
+ ... index=pd.DatetimeIndex(
+ ... [
+ ... "2023-01-01",
+ ... "2023-01-10",
+ ... "2023-01-15",
+ ... "2023-02-01",
+ ... "2023-02-10",
+ ... "2023-02-15",
+ ... ]
+ ... ),
+ ... )
+ >>> ser.resample("MS").sem()
2023-01-01 0.577350
2023-02-01 1.527525
Freq: MS, dtype: float64
@@ -3000,7 +2843,7 @@ def size(self) -> DataFrame | Series:
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b']
+ >>> lst = ["a", "a", "b"]
>>> ser = pd.Series([1, 2, 3], index=lst)
>>> ser
a 1
@@ -3013,8 +2856,9 @@ def size(self) -> DataFrame | Series:
dtype: int64
>>> data = [[1, 2, 3], [1, 5, 6], [7, 8, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["owl", "toucan", "eagle"])
+ >>> df = pd.DataFrame(
+ ... data, columns=["a", "b", "c"], index=["owl", "toucan", "eagle"]
+ ... )
>>> df
a b c
owl 1 2 3
@@ -3028,14 +2872,16 @@ def size(self) -> DataFrame | Series:
For Resampler:
- >>> ser = pd.Series([1, 2, 3], index=pd.DatetimeIndex(
- ... ['2023-01-01', '2023-01-15', '2023-02-01']))
+ >>> ser = pd.Series(
+ ... [1, 2, 3],
+ ... index=pd.DatetimeIndex(["2023-01-01", "2023-01-15", "2023-02-01"]),
+ ... )
>>> ser
2023-01-01 1
2023-01-15 2
2023-02-01 3
dtype: int64
- >>> ser.resample('MS').size()
+ >>> ser.resample("MS").size()
2023-01-01 2
2023-02-01 1
Freq: MS, dtype: int64
@@ -3069,10 +2915,6 @@ def size(self) -> DataFrame | Series:
dtype_backend=dtype_backend,
)
- with com.temp_setattr(self, "as_index", True):
- # size already has the desired behavior in GH#49519, but this makes the
- # as_index=False path of _reindex_output fail on categorical groupers.
- result = self._reindex_output(result, fill_value=0)
if not self.as_index:
# error: Incompatible types in assignment (expression has
# type "DataFrame", variable has type "Series")
@@ -3150,7 +2992,7 @@ def sum(
npfunc=np.sum,
)
- return self._reindex_output(result, fill_value=0)
+ return result
@final
@doc(
@@ -3335,9 +3177,13 @@ def max(
)
@final
- def first(self, numeric_only: bool = False, min_count: int = -1) -> NDFrameT:
+ def first(
+ self, numeric_only: bool = False, min_count: int = -1, skipna: bool = True
+ ) -> NDFrameT:
"""
- Compute the first non-null entry of each column.
+ Compute the first entry of each column within each group.
+
+ Defaults to skipping NA elements.
Parameters
----------
@@ -3345,26 +3191,37 @@ def first(self, numeric_only: bool = False, min_count: int = -1) -> NDFrameT:
Include only float, int, boolean columns.
min_count : int, default -1
The required number of valid values to perform the operation. If fewer
- than ``min_count`` non-NA values are present the result will be NA.
+ than ``min_count`` valid values are present the result will be NA.
+ skipna : bool, default True
+ Exclude NA/null values. If an entire row/column is NA, the result
+ will be NA.
+
+ .. versionadded:: 2.2.1
Returns
-------
Series or DataFrame
- First non-null of values within each group.
+ First values within each group.
See Also
--------
DataFrame.groupby : Apply a function groupby to each row or column of a
DataFrame.
- pandas.core.groupby.DataFrameGroupBy.last : Compute the last non-null entry
+ core.groupby.DataFrameGroupBy.last : Compute the last non-null entry
of each column.
- pandas.core.groupby.DataFrameGroupBy.nth : Take the nth row from each group.
+ core.groupby.DataFrameGroupBy.nth : Take the nth row from each group.
Examples
--------
- >>> df = pd.DataFrame(dict(A=[1, 1, 3], B=[None, 5, 6], C=[1, 2, 3],
- ... D=['3/11/2000', '3/12/2000', '3/13/2000']))
- >>> df['D'] = pd.to_datetime(df['D'])
+ >>> df = pd.DataFrame(
+ ... dict(
+ ... A=[1, 1, 3],
+ ... B=[None, 5, 6],
+ ... C=[1, 2, 3],
+ ... D=["3/11/2000", "3/12/2000", "3/13/2000"],
+ ... )
+ ... )
+ >>> df["D"] = pd.to_datetime(df["D"])
>>> df.groupby("A").first()
B C D
A
@@ -3382,7 +3239,7 @@ def first(self, numeric_only: bool = False, min_count: int = -1) -> NDFrameT:
3 6.0 3
"""
- def first_compat(obj: NDFrameT, axis: AxisInt = 0):
+ def first_compat(obj: NDFrameT):
def first(x: Series):
"""Helper function for first item that isn't NA."""
arr = x.array[notna(x.array)]
@@ -3391,7 +3248,7 @@ def first(x: Series):
return arr[0]
if isinstance(obj, DataFrame):
- return obj.apply(first, axis=axis)
+ return obj.apply(first)
elif isinstance(obj, Series):
return first(obj)
else: # pragma: no cover
@@ -3402,12 +3259,17 @@ def first(x: Series):
min_count=min_count,
alias="first",
npfunc=first_compat,
+ skipna=skipna,
)
@final
- def last(self, numeric_only: bool = False, min_count: int = -1) -> NDFrameT:
+ def last(
+ self, numeric_only: bool = False, min_count: int = -1, skipna: bool = True
+ ) -> NDFrameT:
"""
- Compute the last non-null entry of each column.
+ Compute the last entry of each column within each group.
+
+ Defaults to skipping NA elements.
Parameters
----------
@@ -3416,20 +3278,25 @@ def last(self, numeric_only: bool = False, min_count: int = -1) -> NDFrameT:
everything, then use only numeric data.
min_count : int, default -1
The required number of valid values to perform the operation. If fewer
- than ``min_count`` non-NA values are present the result will be NA.
+ than ``min_count`` valid values are present the result will be NA.
+ skipna : bool, default True
+ Exclude NA/null values. If an entire row/column is NA, the result
+ will be NA.
+
+ .. versionadded:: 2.2.1
Returns
-------
Series or DataFrame
- Last non-null of values within each group.
+ Last of values within each group.
See Also
--------
DataFrame.groupby : Apply a function groupby to each row or column of a
DataFrame.
- pandas.core.groupby.DataFrameGroupBy.first : Compute the first non-null entry
+ core.groupby.DataFrameGroupBy.first : Compute the first non-null entry
of each column.
- pandas.core.groupby.DataFrameGroupBy.nth : Take the nth row from each group.
+ core.groupby.DataFrameGroupBy.nth : Take the nth row from each group.
Examples
--------
@@ -3441,7 +3308,7 @@ def last(self, numeric_only: bool = False, min_count: int = -1) -> NDFrameT:
3 6.0 3
"""
- def last_compat(obj: NDFrameT, axis: AxisInt = 0):
+ def last_compat(obj: NDFrameT):
def last(x: Series):
"""Helper function for last item that isn't NA."""
arr = x.array[notna(x.array)]
@@ -3450,7 +3317,7 @@ def last(x: Series):
return arr[-1]
if isinstance(obj, DataFrame):
- return obj.apply(last, axis=axis)
+ return obj.apply(last)
elif isinstance(obj, Series):
return last(obj)
else: # pragma: no cover
@@ -3461,6 +3328,7 @@ def last(x: Series):
min_count=min_count,
alias="last",
npfunc=last_compat,
+ skipna=skipna,
)
@final
@@ -3480,7 +3348,16 @@ def ohlc(self) -> DataFrame:
For SeriesGroupBy:
- >>> lst = ['SPX', 'CAC', 'SPX', 'CAC', 'SPX', 'CAC', 'SPX', 'CAC',]
+ >>> lst = [
+ ... "SPX",
+ ... "CAC",
+ ... "SPX",
+ ... "CAC",
+ ... "SPX",
+ ... "CAC",
+ ... "SPX",
+ ... "CAC",
+ ... ]
>>> ser = pd.Series([3.4, 9.0, 7.2, 5.2, 8.8, 9.4, 0.1, 0.5], index=lst)
>>> ser
SPX 3.4
@@ -3499,10 +3376,13 @@ def ohlc(self) -> DataFrame:
For DataFrameGroupBy:
- >>> data = {2022: [1.2, 2.3, 8.9, 4.5, 4.4, 3, 2 , 1],
- ... 2023: [3.4, 9.0, 7.2, 5.2, 8.8, 9.4, 8.2, 1.0]}
- >>> df = pd.DataFrame(data, index=['SPX', 'CAC', 'SPX', 'CAC',
- ... 'SPX', 'CAC', 'SPX', 'CAC'])
+ >>> data = {
+ ... 2022: [1.2, 2.3, 8.9, 4.5, 4.4, 3, 2, 1],
+ ... 2023: [3.4, 9.0, 7.2, 5.2, 8.8, 9.4, 8.2, 1.0],
+ ... }
+ >>> df = pd.DataFrame(
+ ... data, index=["SPX", "CAC", "SPX", "CAC", "SPX", "CAC", "SPX", "CAC"]
+ ... )
>>> df
2022 2023
SPX 1.2 3.4
@@ -3521,14 +3401,20 @@ def ohlc(self) -> DataFrame:
For Resampler:
- >>> ser = pd.Series([1, 3, 2, 4, 3, 5],
- ... index=pd.DatetimeIndex(['2023-01-01',
- ... '2023-01-10',
- ... '2023-01-15',
- ... '2023-02-01',
- ... '2023-02-10',
- ... '2023-02-15']))
- >>> ser.resample('MS').ohlc()
+ >>> ser = pd.Series(
+ ... [1, 3, 2, 4, 3, 5],
+ ... index=pd.DatetimeIndex(
+ ... [
+ ... "2023-01-01",
+ ... "2023-01-10",
+ ... "2023-01-15",
+ ... "2023-02-01",
+ ... "2023-02-10",
+ ... "2023-02-15",
+ ... ]
+ ... ),
+ ... )
+ >>> ser.resample("MS").ohlc()
open high low close
2023-01-01 1 3 1 2
2023-02-01 4 5 3 5
@@ -3548,7 +3434,7 @@ def ohlc(self) -> DataFrame:
result = self.obj._constructor_expanddim(
res_values, index=self._grouper.result_index, columns=agg_names
)
- return self._reindex_output(result)
+ return result
result = self._apply_to_column_groupbys(lambda sgb: sgb.ohlc())
return result
@@ -3580,8 +3466,6 @@ def describe(
obj,
not_indexed_same=True,
)
- if self.axis == 1:
- return result.T
# GH#49256 - properly handle the grouping column(s)
result = result.unstack()
@@ -3643,10 +3527,8 @@ def resample(self, rule, *args, include_groups: bool = True, **kwargs) -> Resamp
Examples
--------
- >>> idx = pd.date_range('1/1/2000', periods=4, freq='min')
- >>> df = pd.DataFrame(data=4 * [range(2)],
- ... index=idx,
- ... columns=['a', 'b'])
+ >>> idx = pd.date_range("1/1/2000", periods=4, freq="min")
+ >>> df = pd.DataFrame(data=4 * [range(2)], index=idx, columns=["a", "b"])
>>> df.iloc[2, 0] = 5
>>> df
a b
@@ -3658,7 +3540,7 @@ def resample(self, rule, *args, include_groups: bool = True, **kwargs) -> Resamp
Downsample the DataFrame into 3 minute bins and sum the values of
the timestamps falling into a bin.
- >>> df.groupby('a').resample('3min', include_groups=False).sum()
+ >>> df.groupby("a").resample("3min", include_groups=False).sum()
b
a
0 2000-01-01 00:00:00 2
@@ -3667,7 +3549,7 @@ def resample(self, rule, *args, include_groups: bool = True, **kwargs) -> Resamp
Upsample the series into 30 second bins.
- >>> df.groupby('a').resample('30s', include_groups=False).sum()
+ >>> df.groupby("a").resample("30s", include_groups=False).sum()
b
a
0 2000-01-01 00:00:00 1
@@ -3681,7 +3563,7 @@ def resample(self, rule, *args, include_groups: bool = True, **kwargs) -> Resamp
Resample by month. Values are assigned to the month of the period.
- >>> df.groupby('a').resample('ME', include_groups=False).sum()
+ >>> df.groupby("a").resample("ME", include_groups=False).sum()
b
a
0 2000-01-31 3
@@ -3691,8 +3573,8 @@ def resample(self, rule, *args, include_groups: bool = True, **kwargs) -> Resamp
side of the bin interval.
>>> (
- ... df.groupby('a')
- ... .resample('3min', closed='right', include_groups=False)
+ ... df.groupby("a")
+ ... .resample("3min", closed="right", include_groups=False)
... .sum()
... )
b
@@ -3706,8 +3588,8 @@ def resample(self, rule, *args, include_groups: bool = True, **kwargs) -> Resamp
the left.
>>> (
- ... df.groupby('a')
- ... .resample('3min', closed='right', label='right', include_groups=False)
+ ... df.groupby("a")
+ ... .resample("3min", closed="right", label="right", include_groups=False)
... .sum()
... )
b
@@ -3725,7 +3607,16 @@ def resample(self, rule, *args, include_groups: bool = True, **kwargs) -> Resamp
)
@final
- def rolling(self, *args, **kwargs) -> RollingGroupby:
+ def rolling(
+ self,
+ window: int | datetime.timedelta | str | BaseOffset | BaseIndexer,
+ min_periods: int | None = None,
+ center: bool = False,
+ win_type: str | None = None,
+ on: str | None = None,
+ closed: IntervalClosedType | None = None,
+ method: str = "single",
+ ) -> RollingGroupby:
"""
Return a rolling grouper, providing rolling functionality per group.
@@ -3740,8 +3631,8 @@ def rolling(self, *args, **kwargs) -> RollingGroupby:
If a timedelta, str, or offset, the time period of each window. Each
window will be a variable sized based on the observations included in
the time-period. This is only valid for datetimelike indexes.
- To learn more about the offsets & frequency strings, please see `this link
- `__.
+ To learn more about the offsets & frequency strings, please see
+ :ref:`this link`.
If a BaseIndexer subclass, the window boundaries
based on the defined ``get_window_bounds`` method. Additional rolling
@@ -3780,13 +3671,6 @@ def rolling(self, *args, **kwargs) -> RollingGroupby:
Provided integer column is ignored and excluded from result since
an integer index is not used to calculate the rolling window.
- axis : int or str, default 0
- If ``0`` or ``'index'``, roll across the rows.
-
- If ``1`` or ``'columns'``, roll across the columns.
-
- For `Series` this parameter is unused and defaults to 0.
-
closed : str, default None
If ``'right'``, the first point in the window is excluded from calculations.
@@ -3820,9 +3704,13 @@ def rolling(self, *args, **kwargs) -> RollingGroupby:
Examples
--------
- >>> df = pd.DataFrame({'A': [1, 1, 2, 2],
- ... 'B': [1, 2, 3, 4],
- ... 'C': [0.362, 0.227, 1.267, -0.562]})
+ >>> df = pd.DataFrame(
+ ... {
+ ... "A": [1, 1, 2, 2],
+ ... "B": [1, 2, 3, 4],
+ ... "C": [0.362, 0.227, 1.267, -0.562],
+ ... }
+ ... )
>>> df
A B C
0 1 1 0.362
@@ -3830,7 +3718,7 @@ def rolling(self, *args, **kwargs) -> RollingGroupby:
2 2 3 1.267
3 2 4 -0.562
- >>> df.groupby('A').rolling(2).sum()
+ >>> df.groupby("A").rolling(2).sum()
B C
A
1 0 NaN NaN
@@ -3838,7 +3726,7 @@ def rolling(self, *args, **kwargs) -> RollingGroupby:
2 2 NaN NaN
3 7.0 0.705
- >>> df.groupby('A').rolling(2, min_periods=1).sum()
+ >>> df.groupby("A").rolling(2, min_periods=1).sum()
B C
A
1 0 1.0 0.362
@@ -3846,7 +3734,7 @@ def rolling(self, *args, **kwargs) -> RollingGroupby:
2 2 3.0 1.267
3 7.0 0.705
- >>> df.groupby('A').rolling(2, on='B').sum()
+ >>> df.groupby("A").rolling(2, on="B").sum()
B C
A
1 0 1 NaN
@@ -3858,10 +3746,15 @@ def rolling(self, *args, **kwargs) -> RollingGroupby:
return RollingGroupby(
self._selected_obj,
- *args,
+ window=window,
+ min_periods=min_periods,
+ center=center,
+ win_type=win_type,
+ on=on,
+ closed=closed,
+ method=method,
_grouper=self._grouper,
_as_index=self.as_index,
- **kwargs,
)
@final
@@ -3933,17 +3826,14 @@ def _fill(self, direction: Literal["ffill", "bfill"], limit: int | None = None):
if limit is None:
limit = -1
- ids, _, _ = self._grouper.group_info
- sorted_labels = np.argsort(ids, kind="mergesort").astype(np.intp, copy=False)
- if direction == "bfill":
- sorted_labels = sorted_labels[::-1]
+ ids, ngroups = self._grouper.group_info
col_func = partial(
libgroupby.group_fillna_indexer,
labels=ids,
- sorted_labels=sorted_labels,
limit=limit,
- dropna=self.dropna,
+ compute_ffill=(direction == "ffill"),
+ ngroups=ngroups,
)
def blk_func(values: ArrayLike) -> ArrayLike:
@@ -3980,12 +3870,6 @@ def blk_func(values: ArrayLike) -> ArrayLike:
res_mgr = mgr.apply(blk_func)
new_obj = self._wrap_agged_manager(res_mgr)
-
- if self.axis == 1:
- # Only relevant for DataFrameGroupBy
- new_obj = new_obj.T
- new_obj.columns = self.obj.columns
-
new_obj.index = self.obj.index
return new_obj
@@ -4110,7 +3994,7 @@ def bfill(self, limit: int | None = None):
With Series:
- >>> index = ['Falcon', 'Falcon', 'Parrot', 'Parrot', 'Parrot']
+ >>> index = ["Falcon", "Falcon", "Parrot", "Parrot", "Parrot"]
>>> s = pd.Series([None, 1, None, None, 3], index=index)
>>> s
Falcon NaN
@@ -4136,8 +4020,10 @@ def bfill(self, limit: int | None = None):
With DataFrame:
- >>> df = pd.DataFrame({'A': [1, None, None, None, 4],
- ... 'B': [None, None, 5, None, 7]}, index=index)
+ >>> df = pd.DataFrame(
+ ... {"A": [1, None, None, None, 4], "B": [None, None, 5, None, 7]},
+ ... index=index,
+ ... )
>>> df
A B
Falcon 1.0 NaN
@@ -4198,9 +4084,10 @@ def nth(self) -> GroupByNthSelector:
Examples
--------
- >>> df = pd.DataFrame({'A': [1, 1, 2, 1, 2],
- ... 'B': [np.nan, 2, 3, 4, 5]}, columns=['A', 'B'])
- >>> g = df.groupby('A')
+ >>> df = pd.DataFrame(
+ ... {"A": [1, 1, 2, 1, 2], "B": [np.nan, 2, 3, 4, 5]}, columns=["A", "B"]
+ ... )
+ >>> g = df.groupby("A")
>>> g.nth(0)
A B
0 1 NaN
@@ -4241,7 +4128,7 @@ def nth(self) -> GroupByNthSelector:
Specifying `dropna` allows ignoring ``NaN`` values
- >>> g.nth(0, dropna='any')
+ >>> g.nth(0, dropna="any")
A B
1 1 2.0
2 2 3.0
@@ -4249,7 +4136,7 @@ def nth(self) -> GroupByNthSelector:
When the specified ``n`` is larger than any of the groups, an
empty DataFrame is returned
- >>> g.nth(3, dropna='any')
+ >>> g.nth(3, dropna="any")
Empty DataFrame
Columns: [A, B]
Index: []
@@ -4264,7 +4151,7 @@ def _nth(
if not dropna:
mask = self._make_mask_from_positional_indexer(n)
- ids, _, _ = self._grouper.group_info
+ ids = self._grouper.ids
# Drop NA values in grouping
mask = mask & (ids != -1)
@@ -4287,7 +4174,7 @@ def _nth(
# old behaviour, but with all and any support for DataFrames.
# modified in GH 7559 to have better perf
n = cast(int, n)
- dropped = self._selected_obj.dropna(how=dropna, axis=self.axis)
+ dropped = self._selected_obj.dropna(how=dropna, axis=0)
# get a new grouper for our dropped obj
grouper: np.ndarray | Index | ops.BaseGrouper
@@ -4308,10 +4195,7 @@ def _nth(
values = np.where(nulls, NA, grouper) # type: ignore[call-overload]
grouper = Index(values, dtype="Int64")
- if self.axis == 1:
- grb = dropped.T.groupby(grouper, as_index=self.as_index, sort=self.sort)
- else:
- grb = dropped.groupby(grouper, as_index=self.as_index, sort=self.sort)
+ grb = dropped.groupby(grouper, as_index=self.as_index, sort=self.sort)
return grb.nth(n)
@final
@@ -4352,11 +4236,11 @@ def quantile(
Examples
--------
- >>> df = pd.DataFrame([
- ... ['a', 1], ['a', 2], ['a', 3],
- ... ['b', 1], ['b', 3], ['b', 5]
- ... ], columns=['key', 'val'])
- >>> df.groupby('key').quantile()
+ >>> df = pd.DataFrame(
+ ... [["a", 1], ["a", 2], ["a", 3], ["b", 1], ["b", 3], ["b", 5]],
+ ... columns=["key", "val"],
+ ... )
+ >>> df.groupby("key").quantile()
val
key
a 2.0
@@ -4364,12 +4248,8 @@ def quantile(
"""
mgr = self._get_data_to_aggregate(numeric_only=numeric_only, name="quantile")
obj = self._wrap_agged_manager(mgr)
- if self.axis == 1:
- splitter = self._grouper._get_splitter(obj.T, axis=self.axis)
- sdata = splitter._sorted_data.T
- else:
- splitter = self._grouper._get_splitter(obj, axis=self.axis)
- sdata = splitter._sorted_data
+ splitter = self._grouper._get_splitter(obj)
+ sdata = splitter._sorted_data
starts, ends = lib.generate_slices(splitter._slabels, splitter.ngroups)
@@ -4392,16 +4272,8 @@ def pre_processor(vals: ArrayLike) -> tuple[np.ndarray, DtypeObj | None]:
elif is_bool_dtype(vals.dtype) and isinstance(vals, ExtensionArray):
out = vals.to_numpy(dtype=float, na_value=np.nan)
elif is_bool_dtype(vals.dtype):
- # GH#51424 deprecate to match Series/DataFrame behavior
- warnings.warn(
- f"Allowing bool dtype in {type(self).__name__}.quantile is "
- "deprecated and will raise in a future version, matching "
- "the Series/DataFrame behavior. Cast to uint8 dtype before "
- "calling quantile instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- out = np.asarray(vals)
+ # GH#51424 remove to match Series/DataFrame behavior
+ raise TypeError("Cannot use quantile with bool dtype")
elif needs_i8_conversion(vals.dtype):
inference = vals.dtype
# In this case we need to delay the casting until after the
@@ -4475,7 +4347,10 @@ def post_processor(
qs = np.array([q], dtype=np.float64)
pass_qs = None
- ids, _, ngroups = self._grouper.group_info
+ ids, ngroups = self._grouper.group_info
+ if self.dropna:
+ # splitter drops NA groups, we need to do the same
+ ids = ids[ids >= 0]
nqs = len(qs)
func = partial(
@@ -4607,8 +4482,8 @@ def ngroup(self, ascending: bool = True):
dtype: int64
"""
obj = self._obj_with_exclusions
- index = obj._get_axis(self.axis)
- comp_ids = self._grouper.group_info[0]
+ index = obj.index
+ comp_ids = self._grouper.ids
dtype: type
if self._grouper.has_dropped_na:
@@ -4654,8 +4529,7 @@ def cumcount(self, ascending: bool = True):
Examples
--------
- >>> df = pd.DataFrame([['a'], ['a'], ['a'], ['b'], ['b'], ['a']],
- ... columns=['A'])
+ >>> df = pd.DataFrame([["a"], ["a"], ["a"], ["b"], ["b"], ["a"]], columns=["A"])
>>> df
A
0 a
@@ -4664,7 +4538,7 @@ def cumcount(self, ascending: bool = True):
3 b
4 b
5 a
- >>> df.groupby('A').cumcount()
+ >>> df.groupby("A").cumcount()
0 0
1 1
2 2
@@ -4672,7 +4546,7 @@ def cumcount(self, ascending: bool = True):
4 1
5 3
dtype: int64
- >>> df.groupby('A').cumcount(ascending=False)
+ >>> df.groupby("A").cumcount(ascending=False)
0 3
1 2
2 1
@@ -4681,7 +4555,7 @@ def cumcount(self, ascending: bool = True):
5 0
dtype: int64
"""
- index = self._obj_with_exclusions._get_axis(self.axis)
+ index = self._obj_with_exclusions.index
cumcounts = self._cumcount_array(ascending=ascending)
return self._obj_1d_constructor(cumcounts, index)
@@ -4694,7 +4568,6 @@ def rank(
ascending: bool = True,
na_option: str = "keep",
pct: bool = False,
- axis: AxisInt | lib.NoDefault = lib.no_default,
) -> NDFrameT:
"""
Provide the rank of values within each group.
@@ -4715,12 +4588,6 @@ def rank(
* bottom: smallest rank if descending.
pct : bool, default False
Compute percentage rank of data within each group.
- axis : int, default 0
- The axis of the object over which to compute the rank.
-
- .. deprecated:: 2.1.0
- For axis=1, operate on the underlying object instead. Otherwise
- the axis keyword is not necessary.
Returns
-------
@@ -4746,8 +4613,8 @@ def rank(
7 b 4
8 b 1
9 b 5
- >>> for method in ['average', 'min', 'max', 'dense', 'first']:
- ... df[f'{method}_rank'] = df.groupby('group')['value'].rank(method)
+ >>> for method in ["average", "min", "max", "dense", "first"]:
+ ... df[f"{method}_rank"] = df.groupby("group")["value"].rank(method)
>>> df
group value average_rank min_rank max_rank dense_rank first_rank
0 a 2 1.5 1.0 2.0 1.0 1.0
@@ -4765,43 +4632,34 @@ def rank(
msg = "na_option must be one of 'keep', 'top', or 'bottom'"
raise ValueError(msg)
- if axis is not lib.no_default:
- axis = self.obj._get_axis_number(axis)
- self._deprecate_axis(axis, "rank")
- else:
- axis = 0
-
kwargs = {
"ties_method": method,
"ascending": ascending,
"na_option": na_option,
"pct": pct,
}
- if axis != 0:
- # DataFrame uses different keyword name
- kwargs["method"] = kwargs.pop("ties_method")
- f = lambda x: x.rank(axis=axis, numeric_only=False, **kwargs)
- result = self._python_apply_general(
- f, self._selected_obj, is_transform=True
- )
- return result
return self._cython_transform(
"rank",
numeric_only=False,
- axis=axis,
**kwargs,
)
@final
@Substitution(name="groupby")
@Substitution(see_also=_common_see_also)
- def cumprod(
- self, axis: Axis | lib.NoDefault = lib.no_default, *args, **kwargs
- ) -> NDFrameT:
+ def cumprod(self, *args, **kwargs) -> NDFrameT:
"""
Cumulative product for each group.
+ Parameters
+ ----------
+ *args : tuple
+ Positional arguments to be passed to `func`.
+ **kwargs : dict
+ Additional/specific keyword arguments to be passed to the function,
+ such as `numeric_only` and `skipna`.
+
Returns
-------
Series or DataFrame
@@ -4810,7 +4668,7 @@ def cumprod(
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b']
+ >>> lst = ["a", "a", "b"]
>>> ser = pd.Series([6, 2, 0], index=lst)
>>> ser
a 6
@@ -4826,8 +4684,9 @@ def cumprod(
For DataFrameGroupBy:
>>> data = [[1, 8, 2], [1, 2, 5], [2, 6, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["cow", "horse", "bull"])
+ >>> df = pd.DataFrame(
+ ... data, columns=["a", "b", "c"], index=["cow", "horse", "bull"]
+ ... )
>>> df
a b c
cow 1 8 2
@@ -4842,27 +4701,23 @@ def cumprod(
bull 6 9
"""
nv.validate_groupby_func("cumprod", args, kwargs, ["numeric_only", "skipna"])
- if axis is not lib.no_default:
- axis = self.obj._get_axis_number(axis)
- self._deprecate_axis(axis, "cumprod")
- else:
- axis = 0
-
- if axis != 0:
- f = lambda x: x.cumprod(axis=axis, **kwargs)
- return self._python_apply_general(f, self._selected_obj, is_transform=True)
-
return self._cython_transform("cumprod", **kwargs)
@final
@Substitution(name="groupby")
@Substitution(see_also=_common_see_also)
- def cumsum(
- self, axis: Axis | lib.NoDefault = lib.no_default, *args, **kwargs
- ) -> NDFrameT:
+ def cumsum(self, *args, **kwargs) -> NDFrameT:
"""
Cumulative sum for each group.
+ Parameters
+ ----------
+ *args : tuple
+ Positional arguments to be passed to `func`.
+ **kwargs : dict
+ Additional/specific keyword arguments to be passed to the function,
+ such as `numeric_only` and `skipna`.
+
Returns
-------
Series or DataFrame
@@ -4871,7 +4726,7 @@ def cumsum(
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b']
+ >>> lst = ["a", "a", "b"]
>>> ser = pd.Series([6, 2, 0], index=lst)
>>> ser
a 6
@@ -4887,8 +4742,9 @@ def cumsum(
For DataFrameGroupBy:
>>> data = [[1, 8, 2], [1, 2, 5], [2, 6, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["fox", "gorilla", "lion"])
+ >>> df = pd.DataFrame(
+ ... data, columns=["a", "b", "c"], index=["fox", "gorilla", "lion"]
+ ... )
>>> df
a b c
fox 1 8 2
@@ -4903,16 +4759,6 @@ def cumsum(
lion 6 9
"""
nv.validate_groupby_func("cumsum", args, kwargs, ["numeric_only", "skipna"])
- if axis is not lib.no_default:
- axis = self.obj._get_axis_number(axis)
- self._deprecate_axis(axis, "cumsum")
- else:
- axis = 0
-
- if axis != 0:
- f = lambda x: x.cumsum(axis=axis, **kwargs)
- return self._python_apply_general(f, self._selected_obj, is_transform=True)
-
return self._cython_transform("cumsum", **kwargs)
@final
@@ -4920,13 +4766,20 @@ def cumsum(
@Substitution(see_also=_common_see_also)
def cummin(
self,
- axis: AxisInt | lib.NoDefault = lib.no_default,
numeric_only: bool = False,
**kwargs,
) -> NDFrameT:
"""
Cumulative min for each group.
+ Parameters
+ ----------
+ numeric_only : bool, default False
+ Include only `float`, `int` or `boolean` data.
+ **kwargs : dict, optional
+ Additional keyword arguments to be passed to the function, such as `skipna`,
+ to control whether NA/null values are ignored.
+
Returns
-------
Series or DataFrame
@@ -4935,7 +4788,7 @@ def cummin(
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'a', 'b', 'b', 'b']
+ >>> lst = ["a", "a", "a", "b", "b", "b"]
>>> ser = pd.Series([1, 6, 2, 3, 0, 4], index=lst)
>>> ser
a 1
@@ -4957,8 +4810,9 @@ def cummin(
For DataFrameGroupBy:
>>> data = [[1, 0, 2], [1, 1, 5], [6, 6, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["snake", "rabbit", "turtle"])
+ >>> df = pd.DataFrame(
+ ... data, columns=["a", "b", "c"], index=["snake", "rabbit", "turtle"]
+ ... )
>>> df
a b c
snake 1 0 2
@@ -4973,19 +4827,6 @@ def cummin(
turtle 6 9
"""
skipna = kwargs.get("skipna", True)
- if axis is not lib.no_default:
- axis = self.obj._get_axis_number(axis)
- self._deprecate_axis(axis, "cummin")
- else:
- axis = 0
-
- if axis != 0:
- f = lambda x: np.minimum.accumulate(x, axis)
- obj = self._selected_obj
- if numeric_only:
- obj = obj._get_numeric_data()
- return self._python_apply_general(f, obj, is_transform=True)
-
return self._cython_transform(
"cummin", numeric_only=numeric_only, skipna=skipna
)
@@ -4995,13 +4836,20 @@ def cummin(
@Substitution(see_also=_common_see_also)
def cummax(
self,
- axis: AxisInt | lib.NoDefault = lib.no_default,
numeric_only: bool = False,
**kwargs,
) -> NDFrameT:
"""
Cumulative max for each group.
+ Parameters
+ ----------
+ numeric_only : bool, default False
+ Include only `float`, `int` or `boolean` data.
+ **kwargs : dict, optional
+ Additional keyword arguments to be passed to the function, such as `skipna`,
+ to control whether NA/null values are ignored.
+
Returns
-------
Series or DataFrame
@@ -5010,7 +4858,7 @@ def cummax(
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'a', 'b', 'b', 'b']
+ >>> lst = ["a", "a", "a", "b", "b", "b"]
>>> ser = pd.Series([1, 6, 2, 3, 1, 4], index=lst)
>>> ser
a 1
@@ -5032,8 +4880,9 @@ def cummax(
For DataFrameGroupBy:
>>> data = [[1, 8, 2], [1, 1, 0], [2, 6, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["cow", "horse", "bull"])
+ >>> df = pd.DataFrame(
+ ... data, columns=["a", "b", "c"], index=["cow", "horse", "bull"]
+ ... )
>>> df
a b c
cow 1 8 2
@@ -5048,19 +4897,6 @@ def cummax(
bull 6 9
"""
skipna = kwargs.get("skipna", True)
- if axis is not lib.no_default:
- axis = self.obj._get_axis_number(axis)
- self._deprecate_axis(axis, "cummax")
- else:
- axis = 0
-
- if axis != 0:
- f = lambda x: np.maximum.accumulate(x, axis)
- obj = self._selected_obj
- if numeric_only:
- obj = obj._get_numeric_data()
- return self._python_apply_general(f, obj, is_transform=True)
-
return self._cython_transform(
"cummax", numeric_only=numeric_only, skipna=skipna
)
@@ -5071,7 +4907,6 @@ def shift(
self,
periods: int | Sequence[int] = 1,
freq=None,
- axis: Axis | lib.NoDefault = lib.no_default,
fill_value=lib.no_default,
suffix: str | None = None,
):
@@ -5087,13 +4922,6 @@ def shift(
each period.
freq : str, optional
Frequency string.
- axis : axis to shift, default 0
- Shift direction.
-
- .. deprecated:: 2.1.0
- For axis=1, operate on the underlying object instead. Otherwise
- the axis keyword is not necessary.
-
fill_value : optional
The scalar value to use for newly introduced missing values.
@@ -5118,7 +4946,7 @@ def shift(
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b', 'b']
+ >>> lst = ["a", "a", "b", "b"]
>>> ser = pd.Series([1, 2, 3, 4], index=lst)
>>> ser
a 1
@@ -5136,8 +4964,11 @@ def shift(
For DataFrameGroupBy:
>>> data = [[1, 2, 3], [1, 5, 6], [2, 5, 8], [2, 6, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["tuna", "salmon", "catfish", "goldfish"])
+ >>> df = pd.DataFrame(
+ ... data,
+ ... columns=["a", "b", "c"],
+ ... index=["tuna", "salmon", "catfish", "goldfish"],
+ ... )
>>> df
a b c
tuna 1 2 3
@@ -5151,17 +4982,7 @@ def shift(
catfish NaN NaN
goldfish 5.0 8.0
"""
- if axis is not lib.no_default:
- axis = self.obj._get_axis_number(axis)
- self._deprecate_axis(axis, "shift")
- else:
- axis = 0
-
if is_list_like(periods):
- if axis == 1:
- raise ValueError(
- "If `periods` contains multiple shifts, `axis` cannot be 1."
- )
periods = cast(Sequence, periods)
if len(periods) == 0:
raise ValueError("If `periods` is an iterable, it cannot be empty.")
@@ -5185,9 +5006,12 @@ def shift(
f"Periods must be integer, but {period} is {type(period)}."
)
period = cast(int, period)
- if freq is not None or axis != 0:
+ if freq is not None:
f = lambda x: x.shift(
- period, freq, axis, fill_value # pylint: disable=cell-var-from-loop
+ period, # pylint: disable=cell-var-from-loop
+ freq,
+ 0, # axis
+ fill_value,
)
shifted = self._python_apply_general(
f, self._selected_obj, is_transform=True
@@ -5195,7 +5019,7 @@ def shift(
else:
if fill_value is lib.no_default:
fill_value = None
- ids, _, ngroups = self._grouper.group_info
+ ids, ngroups = self._grouper.group_info
res_indexer = np.zeros(len(ids), dtype=np.int64)
libgroupby.group_shift_indexer(res_indexer, ids, ngroups, period)
@@ -5203,7 +5027,7 @@ def shift(
obj = self._obj_with_exclusions
shifted = obj._reindex_with_indexers(
- {self.axis: (obj.axes[self.axis], res_indexer)},
+ {0: (obj.index, res_indexer)},
fill_value=fill_value,
allow_dups=True,
)
@@ -5226,7 +5050,8 @@ def shift(
@Substitution(name="groupby")
@Substitution(see_also=_common_see_also)
def diff(
- self, periods: int = 1, axis: AxisInt | lib.NoDefault = lib.no_default
+ self,
+ periods: int = 1,
) -> NDFrameT:
"""
First discrete difference of element.
@@ -5238,12 +5063,6 @@ def diff(
----------
periods : int, default 1
Periods to shift for calculating difference, accepts negative values.
- axis : axis to shift, default 0
- Take difference over rows (0) or columns (1).
-
- .. deprecated:: 2.1.0
- For axis=1, operate on the underlying object instead. Otherwise
- the axis keyword is not necessary.
Returns
-------
@@ -5254,7 +5073,7 @@ def diff(
--------
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'a', 'b', 'b', 'b']
+ >>> lst = ["a", "a", "a", "b", "b", "b"]
>>> ser = pd.Series([7, 2, 8, 4, 3, 3], index=lst)
>>> ser
a 7
@@ -5275,9 +5094,10 @@ def diff(
For DataFrameGroupBy:
- >>> data = {'a': [1, 3, 5, 7, 7, 8, 3], 'b': [1, 4, 8, 4, 4, 2, 1]}
- >>> df = pd.DataFrame(data, index=['dog', 'dog', 'dog',
- ... 'mouse', 'mouse', 'mouse', 'mouse'])
+ >>> data = {"a": [1, 3, 5, 7, 7, 8, 3], "b": [1, 4, 8, 4, 4, 2, 1]}
+ >>> df = pd.DataFrame(
+ ... data, index=["dog", "dog", "dog", "mouse", "mouse", "mouse", "mouse"]
+ ... )
>>> df
a b
dog 1 1
@@ -5297,15 +5117,6 @@ def diff(
mouse 1.0 -2.0
mouse -5.0 -1.0
"""
- if axis is not lib.no_default:
- axis = self.obj._get_axis_number(axis)
- self._deprecate_axis(axis, "diff")
- else:
- axis = 0
-
- if axis != 0:
- return self.apply(lambda x: x.diff(periods=periods, axis=axis))
-
obj = self._obj_with_exclusions
shifted = self.shift(periods=periods)
@@ -5328,14 +5139,30 @@ def diff(
def pct_change(
self,
periods: int = 1,
- fill_method: FillnaOptions | None | lib.NoDefault = lib.no_default,
- limit: int | None | lib.NoDefault = lib.no_default,
+ fill_method: None = None,
freq=None,
- axis: Axis | lib.NoDefault = lib.no_default,
):
"""
Calculate pct_change of each value to previous entry in group.
+ Parameters
+ ----------
+ periods : int, default 1
+ Periods to shift for calculating percentage change. Comparing with
+ a period of 1 means adjacent elements are compared, whereas a period
+ of 2 compares every other element.
+
+ fill_method : None
+ Must be None. This argument will be removed in a future version of pandas.
+
+ .. deprecated:: 2.1
+ All options of `fill_method` are deprecated except `fill_method=None`.
+
+ freq : str, pandas offset object, or None, default None
+ The frequency increment for time series data (e.g., 'M' for month-end).
+ If None, the frequency is inferred from the index. Relevant for time
+ series data only.
+
Returns
-------
Series or DataFrame
@@ -5346,7 +5173,7 @@ def pct_change(
For SeriesGroupBy:
- >>> lst = ['a', 'a', 'b', 'b']
+ >>> lst = ["a", "a", "b", "b"]
>>> ser = pd.Series([1, 2, 3, 4], index=lst)
>>> ser
a 1
@@ -5364,8 +5191,11 @@ def pct_change(
For DataFrameGroupBy:
>>> data = [[1, 2, 3], [1, 5, 6], [2, 5, 8], [2, 6, 9]]
- >>> df = pd.DataFrame(data, columns=["a", "b", "c"],
- ... index=["tuna", "salmon", "catfish", "goldfish"])
+ >>> df = pd.DataFrame(
+ ... data,
+ ... columns=["a", "b", "c"],
+ ... index=["tuna", "salmon", "catfish", "goldfish"],
+ ... )
>>> df
a b c
tuna 1 2 3
@@ -5380,62 +5210,26 @@ def pct_change(
goldfish 0.2 0.125
"""
# GH#53491
- if fill_method not in (lib.no_default, None) or limit is not lib.no_default:
- warnings.warn(
- "The 'fill_method' keyword being not None and the 'limit' keyword in "
- f"{type(self).__name__}.pct_change are deprecated and will be removed "
- "in a future version. Either fill in any non-leading NA values prior "
- "to calling pct_change or specify 'fill_method=None' to not fill NA "
- "values.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- if fill_method is lib.no_default:
- if limit is lib.no_default and any(
- grp.isna().values.any() for _, grp in self
- ):
- warnings.warn(
- "The default fill_method='ffill' in "
- f"{type(self).__name__}.pct_change is deprecated and will "
- "be removed in a future version. Either fill in any "
- "non-leading NA values prior to calling pct_change or "
- "specify 'fill_method=None' to not fill NA values.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- fill_method = "ffill"
- if limit is lib.no_default:
- limit = None
-
- if axis is not lib.no_default:
- axis = self.obj._get_axis_number(axis)
- self._deprecate_axis(axis, "pct_change")
- else:
- axis = 0
+ if fill_method is not None:
+ raise ValueError(f"fill_method must be None; got {fill_method=}.")
# TODO(GH#23918): Remove this conditional for SeriesGroupBy when
# GH#23918 is fixed
- if freq is not None or axis != 0:
+ if freq is not None:
f = lambda x: x.pct_change(
periods=periods,
- fill_method=fill_method,
- limit=limit,
freq=freq,
- axis=axis,
+ axis=0,
)
return self._python_apply_general(f, self._selected_obj, is_transform=True)
if fill_method is None: # GH30463
- fill_method = "ffill"
- limit = 0
- filled = getattr(self, fill_method)(limit=limit)
- if self.axis == 0:
- fill_grp = filled.groupby(self._grouper.codes, group_keys=self.group_keys)
+ op = "ffill"
else:
- fill_grp = filled.T.groupby(self._grouper.codes, group_keys=self.group_keys)
+ op = fill_method
+ filled = getattr(self, op)(limit=0)
+ fill_grp = filled.groupby(self._grouper.codes, group_keys=self.group_keys)
shifted = fill_grp.shift(periods=periods, freq=freq)
- if self.axis == 1:
- shifted = shifted.T
return (filled / shifted) - 1
@final
@@ -5463,13 +5257,12 @@ def head(self, n: int = 5) -> NDFrameT:
Examples
--------
- >>> df = pd.DataFrame([[1, 2], [1, 4], [5, 6]],
- ... columns=['A', 'B'])
- >>> df.groupby('A').head(1)
+ >>> df = pd.DataFrame([[1, 2], [1, 4], [5, 6]], columns=["A", "B"])
+ >>> df.groupby("A").head(1)
A B
0 1 2
2 5 6
- >>> df.groupby('A').head(-1)
+ >>> df.groupby("A").head(-1)
A B
0 1 2
"""
@@ -5501,13 +5294,14 @@ def tail(self, n: int = 5) -> NDFrameT:
Examples
--------
- >>> df = pd.DataFrame([['a', 1], ['a', 2], ['b', 1], ['b', 2]],
- ... columns=['A', 'B'])
- >>> df.groupby('A').tail(1)
+ >>> df = pd.DataFrame(
+ ... [["a", 1], ["a", 2], ["b", 1], ["b", 2]], columns=["A", "B"]
+ ... )
+ >>> df.groupby("A").tail(1)
A B
1 a 2
3 b 2
- >>> df.groupby('A').tail(-1)
+ >>> df.groupby("A").tail(-1)
A B
1 a 2
3 b 2
@@ -5522,7 +5316,7 @@ def tail(self, n: int = 5) -> NDFrameT:
@final
def _mask_selected_obj(self, mask: npt.NDArray[np.bool_]) -> NDFrameT:
"""
- Return _selected_obj with mask applied to the correct axis.
+ Return _selected_obj with mask applied.
Parameters
----------
@@ -5534,111 +5328,9 @@ def _mask_selected_obj(self, mask: npt.NDArray[np.bool_]) -> NDFrameT:
Series or DataFrame
Filtered _selected_obj.
"""
- ids = self._grouper.group_info[0]
+ ids = self._grouper.ids
mask = mask & (ids != -1)
-
- if self.axis == 0:
- return self._selected_obj[mask]
- else:
- return self._selected_obj.iloc[:, mask]
-
- @final
- def _reindex_output(
- self,
- output: OutputFrameOrSeries,
- fill_value: Scalar = np.nan,
- qs: npt.NDArray[np.float64] | None = None,
- ) -> OutputFrameOrSeries:
- """
- If we have categorical groupers, then we might want to make sure that
- we have a fully re-indexed output to the levels. This means expanding
- the output space to accommodate all values in the cartesian product of
- our groups, regardless of whether they were observed in the data or
- not. This will expand the output space if there are missing groups.
-
- The method returns early without modifying the input if the number of
- groupings is less than 2, self.observed == True or none of the groupers
- are categorical.
-
- Parameters
- ----------
- output : Series or DataFrame
- Object resulting from grouping and applying an operation.
- fill_value : scalar, default np.nan
- Value to use for unobserved categories if self.observed is False.
- qs : np.ndarray[float64] or None, default None
- quantile values, only relevant for quantile.
-
- Returns
- -------
- Series or DataFrame
- Object (potentially) re-indexed to include all possible groups.
- """
- groupings = self._grouper.groupings
- if len(groupings) == 1:
- return output
-
- # if we only care about the observed values
- # we are done
- elif self.observed:
- return output
-
- # reindexing only applies to a Categorical grouper
- elif not any(
- isinstance(ping.grouping_vector, (Categorical, CategoricalIndex))
- for ping in groupings
- ):
- return output
-
- levels_list = [ping._group_index for ping in groupings]
- names = self._grouper.names
- if qs is not None:
- # error: Argument 1 to "append" of "list" has incompatible type
- # "ndarray[Any, dtype[floating[_64Bit]]]"; expected "Index"
- levels_list.append(qs) # type: ignore[arg-type]
- names = names + [None]
- index = MultiIndex.from_product(levels_list, names=names)
- if self.sort:
- index = index.sort_values()
-
- if self.as_index:
- # Always holds for SeriesGroupBy unless GH#36507 is implemented
- d = {
- self.obj._get_axis_name(self.axis): index,
- "copy": False,
- "fill_value": fill_value,
- }
- return output.reindex(**d) # type: ignore[arg-type]
-
- # GH 13204
- # Here, the categorical in-axis groupers, which need to be fully
- # expanded, are columns in `output`. An idea is to do:
- # output = output.set_index(self._grouper.names)
- # .reindex(index).reset_index()
- # but special care has to be taken because of possible not-in-axis
- # groupers.
- # So, we manually select and drop the in-axis grouper columns,
- # reindex `output`, and then reset the in-axis grouper columns.
-
- # Select in-axis groupers
- in_axis_grps = [
- (i, ping.name) for (i, ping) in enumerate(groupings) if ping.in_axis
- ]
- if len(in_axis_grps) > 0:
- g_nums, g_names = zip(*in_axis_grps)
- output = output.drop(labels=list(g_names), axis=1)
-
- # Set a temp index and reindex (possibly expanding)
- output = output.set_index(self._grouper.result_index).reindex(
- index, copy=False, fill_value=fill_value
- )
-
- # Reset in-axis grouper columns
- # (using level numbers `g_nums` because level names may not be unique)
- if len(in_axis_grps) > 0:
- output = output.reset_index(level=g_nums)
-
- return output.reset_index(drop=True)
+ return self._selected_obj[mask]
@final
def sample(
@@ -5739,13 +5431,11 @@ def sample(
return self._selected_obj
size = sample.process_sampling_size(n, frac, replace)
if weights is not None:
- weights_arr = sample.preprocess_weights(
- self._selected_obj, weights, axis=self.axis
- )
+ weights_arr = sample.preprocess_weights(self._selected_obj, weights, axis=0)
random_state = com.random_state(random_state)
- group_iterator = self._grouper.get_iterator(self._selected_obj, self.axis)
+ group_iterator = self._grouper.get_iterator(self._selected_obj)
sampled_indices = []
for labels, obj in group_iterator:
@@ -5767,13 +5457,12 @@ def sample(
sampled_indices.append(grp_indices[grp_sample])
sampled_indices = np.concatenate(sampled_indices)
- return self._selected_obj.take(sampled_indices, axis=self.axis)
+ return self._selected_obj.take(sampled_indices, axis=0)
def _idxmax_idxmin(
self,
how: Literal["idxmax", "idxmin"],
ignore_unobserved: bool = False,
- axis: Axis | None | lib.NoDefault = lib.no_default,
skipna: bool = True,
numeric_only: bool = False,
) -> NDFrameT:
@@ -5783,9 +5472,6 @@ def _idxmax_idxmin(
----------
how : {'idxmin', 'idxmax'}
Whether to compute idxmin or idxmax.
- axis : {{0 or 'index', 1 or 'columns'}}, default None
- The axis to use. 0 or 'index' for row-wise, 1 or 'columns' for column-wise.
- If axis is not provided, grouper's axis is used.
numeric_only : bool, default False
Include only float, int, boolean columns.
skipna : bool, default True
@@ -5800,25 +5486,13 @@ def _idxmax_idxmin(
Series or DataFrame
idxmax or idxmin for the groupby operation.
"""
- if axis is not lib.no_default:
- if axis is None:
- axis = self.axis
- axis = self.obj._get_axis_number(axis)
- self._deprecate_axis(axis, how)
- else:
- axis = self.axis
-
if not self.observed and any(
ping._passed_categorical for ping in self._grouper.groupings
):
- expected_len = np.prod(
- [len(ping._group_index) for ping in self._grouper.groupings]
- )
- if len(self._grouper.groupings) == 1:
- result_len = len(self._grouper.groupings[0].grouping_vector.unique())
- else:
- # result_index only contains observed groups in this case
- result_len = len(self._grouper.result_index)
+ expected_len = len(self._grouper.result_index)
+ # TODO: Better way to find # of observed groups?
+ group_sizes = self._grouper.size()
+ result_len = group_sizes[group_sizes > 0].shape[0]
assert result_len <= expected_len
has_unobserved = result_len < expected_len
@@ -5837,36 +5511,11 @@ def _idxmax_idxmin(
f"Can't get {how} of an empty group due to unobserved categories. "
"Specify observed=True in groupby instead."
)
- elif not skipna:
- if self._obj_with_exclusions.isna().any(axis=None):
- warnings.warn(
- f"The behavior of {type(self).__name__}.{how} with all-NA "
- "values, or any-NA and skipna=False, is deprecated. In a future "
- "version this will raise ValueError",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- if axis == 1:
- try:
-
- def func(df):
- method = getattr(df, how)
- return method(axis=axis, skipna=skipna, numeric_only=numeric_only)
-
- func.__name__ = how
- result = self._python_apply_general(
- func, self._obj_with_exclusions, not_indexed_same=True
- )
- except ValueError as err:
- name = "argmax" if how == "idxmax" else "argmin"
- if f"attempt to get {name} of an empty sequence" in str(err):
- raise ValueError(
- f"Can't get {how} of an empty group due to unobserved "
- "categories. Specify observed=True in groupby instead."
- ) from None
- raise
- return result
+ elif not skipna and self._obj_with_exclusions.isna().any(axis=None):
+ raise ValueError(
+ f"{type(self).__name__}.{how} with skipna=False encountered an NA "
+ f"value."
+ )
result = self._agg_general(
numeric_only=numeric_only,
@@ -5877,7 +5526,7 @@ def func(df):
return result
def _wrap_idxmax_idxmin(self, res: NDFrameT) -> NDFrameT:
- index = self.obj._get_axis(self.axis)
+ index = self.obj.index
if res.size == 0:
result = res.astype(index.dtype)
else:
@@ -5908,7 +5557,6 @@ def _wrap_idxmax_idxmin(self, res: NDFrameT) -> NDFrameT:
def get_groupby(
obj: NDFrame,
by: _KeysArgType | None = None,
- axis: AxisInt = 0,
grouper: ops.BaseGrouper | None = None,
group_keys: bool = True,
) -> GroupBy:
@@ -5927,7 +5575,6 @@ def get_groupby(
return klass(
obj=obj,
keys=by,
- axis=axis,
grouper=grouper,
group_keys=group_keys,
)
@@ -5956,7 +5603,7 @@ def _insert_quantile_level(idx: Index, qs: npt.NDArray[np.float64]) -> MultiInde
idx = cast(MultiIndex, idx)
levels = list(idx.levels) + [lev]
codes = [np.repeat(x, nqs) for x in idx.codes] + [np.tile(lev_codes, len(idx))]
- mi = MultiIndex(levels=levels, codes=codes, names=idx.names + [None])
+ mi = MultiIndex(levels=levels, codes=codes, names=list(idx.names) + [None])
else:
nidx = len(idx)
idx_codes = coerce_indexer_dtype(np.arange(nidx), idx)
diff --git a/pandas/core/groupby/grouper.py b/pandas/core/groupby/grouper.py
index e2224caad9e84..239d78b3b8b7a 100644
--- a/pandas/core/groupby/grouper.py
+++ b/pandas/core/groupby/grouper.py
@@ -2,26 +2,19 @@
Provide user facing operators for doing the split part of the
split-apply-combine paradigm.
"""
+
from __future__ import annotations
from typing import (
TYPE_CHECKING,
final,
)
-import warnings
import numpy as np
-from pandas._config import (
- using_copy_on_write,
- warn_copy_on_write,
-)
-
-from pandas._libs import lib
from pandas._libs.tslibs import OutOfBoundsDatetime
from pandas.errors import InvalidIndexError
from pandas.util._decorators import cache_readonly
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.common import (
is_list_like,
@@ -39,7 +32,6 @@
from pandas.core.groupby import ops
from pandas.core.groupby.categorical import recode_for_groupby
from pandas.core.indexes.api import (
- CategoricalIndex,
Index,
MultiIndex,
)
@@ -55,7 +47,6 @@
from pandas._typing import (
ArrayLike,
- Axis,
NDFrameT,
npt,
)
@@ -68,14 +59,18 @@ class Grouper:
A Grouper allows the user to specify a groupby instruction for an object.
This specification will select a column via the key parameter, or if the
- level and/or axis parameters are given, a level of the index of the target
+ level parameter is given, a level of the index of the target
object.
- If `axis` and/or `level` are passed as keywords to both `Grouper` and
+ If ``level`` is passed as a keyword to both `Grouper` and
`groupby`, the values passed to `Grouper` take precedence.
Parameters
----------
+ *args
+ Currently unused, reserved for future use.
+ **kwargs
+ Dictionary of the keyword arguments to pass to Grouper.
key : str, defaults to None
Groupby key, which selects the grouping column of the target.
level : name/number, defaults to None
@@ -83,10 +78,7 @@ class Grouper:
freq : str / frequency object, defaults to None
This will groupby the specified frequency if the target selection
(via key or level) is a datetime-like object. For full specification
- of available frequencies, please see `here
- `_.
- axis : str, int, defaults to 0
- Number/name of the axis.
+ of available frequencies, please see :ref:`here`.
sort : bool, default to False
Whether to sort the resulting labels.
closed : {'left' or 'right'}
@@ -151,15 +143,15 @@ class Grouper:
Specify a resample operation on the column 'Publish date'
>>> df = pd.DataFrame(
- ... {
- ... "Publish date": [
+ ... {
+ ... "Publish date": [
... pd.Timestamp("2000-01-02"),
... pd.Timestamp("2000-01-02"),
... pd.Timestamp("2000-01-09"),
- ... pd.Timestamp("2000-01-16")
+ ... pd.Timestamp("2000-01-16"),
... ],
... "ID": [0, 1, 2, 3],
- ... "Price": [10, 20, 30, 40]
+ ... "Price": [10, 20, 30, 40],
... }
... )
>>> df
@@ -177,8 +169,8 @@ class Grouper:
If you want to adjust the start of the bins based on a fixed timestamp:
- >>> start, end = '2000-10-01 23:30:00', '2000-10-02 00:30:00'
- >>> rng = pd.date_range(start, end, freq='7min')
+ >>> start, end = "2000-10-01 23:30:00", "2000-10-02 00:30:00"
+ >>> rng = pd.date_range(start, end, freq="7min")
>>> ts = pd.Series(np.arange(len(rng)) * 3, index=rng)
>>> ts
2000-10-01 23:30:00 0
@@ -192,7 +184,7 @@ class Grouper:
2000-10-02 00:26:00 24
Freq: 7min, dtype: int64
- >>> ts.groupby(pd.Grouper(freq='17min')).sum()
+ >>> ts.groupby(pd.Grouper(freq="17min")).sum()
2000-10-01 23:14:00 0
2000-10-01 23:31:00 9
2000-10-01 23:48:00 21
@@ -200,7 +192,7 @@ class Grouper:
2000-10-02 00:22:00 24
Freq: 17min, dtype: int64
- >>> ts.groupby(pd.Grouper(freq='17min', origin='epoch')).sum()
+ >>> ts.groupby(pd.Grouper(freq="17min", origin="epoch")).sum()
2000-10-01 23:18:00 0
2000-10-01 23:35:00 18
2000-10-01 23:52:00 27
@@ -208,7 +200,7 @@ class Grouper:
2000-10-02 00:26:00 24
Freq: 17min, dtype: int64
- >>> ts.groupby(pd.Grouper(freq='17min', origin='2000-01-01')).sum()
+ >>> ts.groupby(pd.Grouper(freq="17min", origin="2000-01-01")).sum()
2000-10-01 23:24:00 3
2000-10-01 23:41:00 15
2000-10-01 23:58:00 45
@@ -218,14 +210,14 @@ class Grouper:
If you want to adjust the start of the bins with an `offset` Timedelta, the two
following lines are equivalent:
- >>> ts.groupby(pd.Grouper(freq='17min', origin='start')).sum()
+ >>> ts.groupby(pd.Grouper(freq="17min", origin="start")).sum()
2000-10-01 23:30:00 9
2000-10-01 23:47:00 21
2000-10-02 00:04:00 54
2000-10-02 00:21:00 24
Freq: 17min, dtype: int64
- >>> ts.groupby(pd.Grouper(freq='17min', offset='23h30min')).sum()
+ >>> ts.groupby(pd.Grouper(freq="17min", offset="23h30min")).sum()
2000-10-01 23:30:00 9
2000-10-01 23:47:00 21
2000-10-02 00:04:00 54
@@ -235,7 +227,7 @@ class Grouper:
To replace the use of the deprecated `base` argument, you can now use `offset`,
in this example it is equivalent to have `base=2`:
- >>> ts.groupby(pd.Grouper(freq='17min', offset='2min')).sum()
+ >>> ts.groupby(pd.Grouper(freq="17min", offset="2min")).sum()
2000-10-01 23:16:00 0
2000-10-01 23:33:00 9
2000-10-01 23:50:00 36
@@ -246,10 +238,9 @@ class Grouper:
sort: bool
dropna: bool
- _gpr_index: Index | None
_grouper: Index | None
- _attributes: tuple[str, ...] = ("key", "level", "freq", "axis", "sort", "dropna")
+ _attributes: tuple[str, ...] = ("key", "level", "freq", "sort", "dropna")
def __new__(cls, *args, **kwargs):
if kwargs.get("freq") is not None:
@@ -263,36 +254,17 @@ def __init__(
key=None,
level=None,
freq=None,
- axis: Axis | lib.NoDefault = lib.no_default,
sort: bool = False,
dropna: bool = True,
) -> None:
- if type(self) is Grouper:
- # i.e. not TimeGrouper
- if axis is not lib.no_default:
- warnings.warn(
- "Grouper axis keyword is deprecated and will be removed in a "
- "future version. To group on axis=1, use obj.T.groupby(...) "
- "instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- axis = 0
- if axis is lib.no_default:
- axis = 0
-
self.key = key
self.level = level
self.freq = freq
- self.axis = axis
self.sort = sort
self.dropna = dropna
self._grouper_deprecated = None
self._indexer_deprecated: npt.NDArray[np.intp] | None = None
- self._obj_deprecated = None
- self._gpr_index = None
self.binner = None
self._grouper = None
self._indexer: npt.NDArray[np.intp] | None = None
@@ -315,7 +287,6 @@ def _get_grouper(
grouper, _, obj = get_grouper(
obj,
[self.key],
- axis=self.axis,
level=self.level,
sort=self.sort,
validate=validate,
@@ -381,7 +352,7 @@ def _set_grouper(
ax = Index(obj[key], name=key)
else:
- ax = obj._get_axis(self.axis)
+ ax = obj.index
if self.level is not None:
level = self.level
@@ -404,79 +375,14 @@ def _set_grouper(
kind="mergesort", na_position="first"
)
ax = ax.take(indexer)
- obj = obj.take(indexer, axis=self.axis)
+ obj = obj.take(indexer, axis=0)
- # error: Incompatible types in assignment (expression has type
- # "NDFrameT", variable has type "None")
- self._obj_deprecated = obj # type: ignore[assignment]
- self._gpr_index = ax
return obj, ax, indexer
- @final
- @property
- def ax(self) -> Index:
- warnings.warn(
- f"{type(self).__name__}.ax is deprecated and will be removed in a "
- "future version. Use Resampler.ax instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- index = self._gpr_index
- if index is None:
- raise ValueError("_set_grouper must be called before ax is accessed")
- return index
-
- @final
- @property
- def indexer(self):
- warnings.warn(
- f"{type(self).__name__}.indexer is deprecated and will be removed "
- "in a future version. Use Resampler.indexer instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self._indexer_deprecated
-
- @final
- @property
- def obj(self):
- # TODO(3.0): enforcing these deprecations on Grouper should close
- # GH#25564, GH#41930
- warnings.warn(
- f"{type(self).__name__}.obj is deprecated and will be removed "
- "in a future version. Use GroupBy.indexer instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self._obj_deprecated
-
- @final
- @property
- def grouper(self):
- warnings.warn(
- f"{type(self).__name__}.grouper is deprecated and will be removed "
- "in a future version. Use GroupBy.grouper instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self._grouper_deprecated
-
- @final
- @property
- def groups(self):
- warnings.warn(
- f"{type(self).__name__}.groups is deprecated and will be removed "
- "in a future version. Use GroupBy.groups instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- # error: "None" has no attribute "groups"
- return self._grouper_deprecated.groups # type: ignore[attr-defined]
-
@final
def __repr__(self) -> str:
attrs_list = (
- f"{attr_name}={repr(getattr(self, attr_name))}"
+ f"{attr_name}={getattr(self, attr_name)!r}"
for attr_name in self._attributes
if getattr(self, attr_name) is not None
)
@@ -520,7 +426,6 @@ class Grouping:
"""
_codes: npt.NDArray[np.signedinteger] | None = None
- _all_grouper: Categorical | None
_orig_cats: Index | None
_index: Index
@@ -539,7 +444,6 @@ def __init__(
self.level = level
self._orig_grouper = grouper
grouping_vector = _convert_grouper(index, grouper)
- self._all_grouper = None
self._orig_cats = None
self._index = index
self._sort = sort
@@ -623,9 +527,7 @@ def __init__(
elif isinstance(getattr(grouping_vector, "dtype", None), CategoricalDtype):
# a passed Categorical
self._orig_cats = grouping_vector.categories
- grouping_vector, self._all_grouper = recode_for_groupby(
- grouping_vector, sort, observed
- )
+ grouping_vector = recode_for_groupby(grouping_vector, sort, observed)
self.grouping_vector = grouping_vector
@@ -675,7 +577,7 @@ def _ilevel(self) -> int | None:
@property
def ngroups(self) -> int:
- return len(self._group_index)
+ return len(self.uniques)
@cache_readonly
def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
@@ -690,89 +592,9 @@ def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
def codes(self) -> npt.NDArray[np.signedinteger]:
return self._codes_and_uniques[0]
- @cache_readonly
- def _group_arraylike(self) -> ArrayLike:
- """
- Analogous to result_index, but holding an ArrayLike to ensure
- we can retain ExtensionDtypes.
- """
- if self._all_grouper is not None:
- # retain dtype for categories, including unobserved ones
- return self._result_index._values
-
- elif self._passed_categorical:
- return self._group_index._values
-
- return self._codes_and_uniques[1]
-
- @property
- def group_arraylike(self) -> ArrayLike:
- """
- Analogous to result_index, but holding an ArrayLike to ensure
- we can retain ExtensionDtypes.
- """
- warnings.warn(
- "group_arraylike is deprecated and will be removed in a future "
- "version of pandas",
- category=FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self._group_arraylike
-
- @cache_readonly
- def _result_index(self) -> Index:
- # result_index retains dtype for categories, including unobserved ones,
- # which group_index does not
- if self._all_grouper is not None:
- group_idx = self._group_index
- assert isinstance(group_idx, CategoricalIndex)
- cats = self._orig_cats
- # set_categories is dynamically added
- return group_idx.set_categories(cats) # type: ignore[attr-defined]
- return self._group_index
-
@property
- def result_index(self) -> Index:
- warnings.warn(
- "result_index is deprecated and will be removed in a future "
- "version of pandas",
- category=FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self._result_index
-
- @cache_readonly
- def _group_index(self) -> Index:
- codes, uniques = self._codes_and_uniques
- if not self._dropna and self._passed_categorical:
- assert isinstance(uniques, Categorical)
- if self._sort and (codes == len(uniques)).any():
- # Add NA value on the end when sorting
- uniques = Categorical.from_codes(
- np.append(uniques.codes, [-1]), uniques.categories, validate=False
- )
- elif len(codes) > 0:
- # Need to determine proper placement of NA value when not sorting
- cat = self.grouping_vector
- na_idx = (cat.codes < 0).argmax()
- if cat.codes[na_idx] < 0:
- # count number of unique codes that comes before the nan value
- na_unique_idx = algorithms.nunique_ints(cat.codes[:na_idx])
- new_codes = np.insert(uniques.codes, na_unique_idx, -1)
- uniques = Categorical.from_codes(
- new_codes, uniques.categories, validate=False
- )
- return Index._with_infer(uniques, name=self.name)
-
- @property
- def group_index(self) -> Index:
- warnings.warn(
- "group_index is deprecated and will be removed in a future "
- "version of pandas",
- category=FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self._group_index
+ def uniques(self) -> ArrayLike:
+ return self._codes_and_uniques[1]
@cache_readonly
def _codes_and_uniques(self) -> tuple[npt.NDArray[np.signedinteger], ArrayLike]:
@@ -792,29 +614,31 @@ def _codes_and_uniques(self) -> tuple[npt.NDArray[np.signedinteger], ArrayLike]:
else:
ucodes = np.arange(len(categories))
- uniques = Categorical.from_codes(
- codes=ucodes, categories=categories, ordered=cat.ordered, validate=False
- )
-
- codes = cat.codes
+ has_dropped_na = False
if not self._dropna:
- na_mask = codes < 0
+ na_mask = cat.isna()
if np.any(na_mask):
+ has_dropped_na = True
if self._sort:
- # Replace NA codes with `largest code + 1`
+ # NA goes at the end, gets `largest non-NA code + 1`
na_code = len(categories)
- codes = np.where(na_mask, na_code, codes)
else:
- # Insert NA code into the codes based on first appearance
- # A negative code must exist, no need to check codes[na_idx] < 0
+ # Insert NA in result based on first appearance, need
+ # the number of unique codes prior
na_idx = na_mask.argmax()
- # count number of unique codes that comes before the nan value
- na_code = algorithms.nunique_ints(codes[:na_idx])
- codes = np.where(codes >= na_code, codes + 1, codes)
- codes = np.where(na_mask, na_code, codes)
+ na_code = algorithms.nunique_ints(cat.codes[:na_idx])
+ ucodes = np.insert(ucodes, na_code, -1)
+
+ uniques = Categorical.from_codes(
+ codes=ucodes, categories=categories, ordered=cat.ordered, validate=False
+ )
+ codes = cat.codes
- if not self._observed:
- uniques = uniques.reorder_categories(self._orig_cats)
+ if has_dropped_na:
+ if not self._sort:
+ # NA code is based on first appearance, increment higher codes
+ codes = np.where(codes >= na_code, codes + 1, codes)
+ codes = np.where(na_mask, na_code, codes)
return codes, uniques
@@ -838,15 +662,16 @@ def _codes_and_uniques(self) -> tuple[npt.NDArray[np.signedinteger], ArrayLike]:
return codes, uniques
@cache_readonly
- def groups(self) -> dict[Hashable, np.ndarray]:
- cats = Categorical.from_codes(self.codes, self._group_index, validate=False)
+ def groups(self) -> dict[Hashable, Index]:
+ codes, uniques = self._codes_and_uniques
+ uniques = Index._with_infer(uniques, name=self.name)
+ cats = Categorical.from_codes(codes, uniques, validate=False)
return self._index.groupby(cats)
def get_grouper(
obj: NDFrameT,
key=None,
- axis: Axis = 0,
level=None,
sort: bool = True,
observed: bool = False,
@@ -862,8 +687,8 @@ def get_grouper(
Groupers are ultimately index mappings. They can originate as:
index mappings, keys to columns, functions, or Groupers
- Groupers enable local references to axis,level,sort, while
- the passed in axis, level, and sort are 'global'.
+ Groupers enable local references to level,sort, while
+ the passed in level, and sort are 'global'.
This routine tries to figure out what the passing in references
are and then creates a Grouping for each one, combined into
@@ -875,10 +700,10 @@ def get_grouper(
If validate, then check for key/level overlaps.
"""
- group_axis = obj._get_axis(axis)
+ group_axis = obj.index
# validate that the passed single level is compatible with the passed
- # axis of the object
+ # index of the object
if level is not None:
# TODO: These if-block and else-block are almost same.
# MultiIndex instance check is removable, but it seems that there are
@@ -911,11 +736,8 @@ def get_grouper(
raise ValueError("multiple levels only valid with MultiIndex")
if isinstance(level, str):
- if obj._get_axis(axis).name != level:
- raise ValueError(
- f"level name {level} is not the name "
- f"of the {obj._get_axis_name(axis)}"
- )
+ if obj.index.name != level:
+ raise ValueError(f"level name {level} is not the name of the index")
elif level > 0 or level < -1:
raise ValueError("level > 0 or level < -1 only valid with MultiIndex")
@@ -999,28 +821,15 @@ def is_in_axis(key) -> bool:
def is_in_obj(gpr) -> bool:
if not hasattr(gpr, "name"):
return False
- if using_copy_on_write() or warn_copy_on_write():
- # For the CoW case, we check the references to determine if the
- # series is part of the object
- try:
- obj_gpr_column = obj[gpr.name]
- except (KeyError, IndexError, InvalidIndexError, OutOfBoundsDatetime):
- return False
- if isinstance(gpr, Series) and isinstance(obj_gpr_column, Series):
- return gpr._mgr.references_same_values( # type: ignore[union-attr]
- obj_gpr_column._mgr, 0 # type: ignore[arg-type]
- )
- return False
+ # We check the references to determine if the
+ # series is part of the object
try:
- return gpr is obj[gpr.name]
+ obj_gpr_column = obj[gpr.name]
except (KeyError, IndexError, InvalidIndexError, OutOfBoundsDatetime):
- # IndexError reached in e.g. test_skip_group_keys when we pass
- # lambda here
- # InvalidIndexError raised on key-types inappropriate for index,
- # e.g. DatetimeIndex.get_loc(tuple())
- # OutOfBoundsDatetime raised when obj is a Series with DatetimeIndex
- # and gpr.name is month str
return False
+ if isinstance(gpr, Series) and isinstance(obj_gpr_column, Series):
+ return gpr._mgr.references_same_values(obj_gpr_column._mgr, 0)
+ return False
for gpr, level in zip(keys, levels):
if is_in_obj(gpr): # df.groupby(df['name'])
@@ -1030,14 +839,14 @@ def is_in_obj(gpr) -> bool:
elif is_in_axis(gpr): # df.groupby('name')
if obj.ndim != 1 and gpr in obj:
if validate:
- obj._check_label_or_level_ambiguity(gpr, axis=axis)
+ obj._check_label_or_level_ambiguity(gpr, axis=0)
in_axis, name, gpr = True, gpr, obj[gpr]
if gpr.ndim != 1:
# non-unique columns; raise here to get the name in the
# exception message
raise ValueError(f"Grouper for '{name}' not 1-dimensional")
exclusions.add(name)
- elif obj._is_level_reference(gpr, axis=axis):
+ elif obj._is_level_reference(gpr, axis=0):
in_axis, level, gpr = False, gpr, None
else:
raise KeyError(gpr)
diff --git a/pandas/core/groupby/indexing.py b/pandas/core/groupby/indexing.py
index a3c5ab8edc94e..c658f625d5ea9 100644
--- a/pandas/core/groupby/indexing.py
+++ b/pandas/core/groupby/indexing.py
@@ -99,8 +99,9 @@ def _positional_selector(self) -> GroupByPositionalSelector:
Examples
--------
- >>> df = pd.DataFrame([["a", 1], ["a", 2], ["a", 3], ["b", 4], ["b", 5]],
- ... columns=["A", "B"])
+ >>> df = pd.DataFrame(
+ ... [["a", 1], ["a", 2], ["a", 3], ["b", 4], ["b", 5]], columns=["A", "B"]
+ ... )
>>> df.groupby("A")._positional_selector[1:2]
A B
1 a 2
@@ -113,7 +114,6 @@ def _positional_selector(self) -> GroupByPositionalSelector:
4 b 5
"""
if TYPE_CHECKING:
- # pylint: disable-next=used-before-assignment
groupby_self = cast(groupby.GroupBy, self)
else:
groupby_self = self
diff --git a/pandas/core/groupby/numba_.py b/pandas/core/groupby/numba_.py
index 3b7a58e87603e..b22fc9248eeca 100644
--- a/pandas/core/groupby/numba_.py
+++ b/pandas/core/groupby/numba_.py
@@ -1,4 +1,5 @@
"""Common utilities for Numba operations with groupby ops"""
+
from __future__ import annotations
import functools
diff --git a/pandas/core/groupby/ops.py b/pandas/core/groupby/ops.py
index 5e83eaee02afc..8585ae3828247 100644
--- a/pandas/core/groupby/ops.py
+++ b/pandas/core/groupby/ops.py
@@ -5,6 +5,7 @@
operations, primarily in cython. These classes (BaseGrouper and BinGrouper)
are contained *in* the SeriesGroupBy and DataFrameGroupBy objects.
"""
+
from __future__ import annotations
import collections
@@ -49,6 +50,7 @@
maybe_fill,
)
+from pandas.core.arrays import Categorical
from pandas.core.frame import DataFrame
from pandas.core.groupby import grouper
from pandas.core.indexes.api import (
@@ -61,7 +63,6 @@
from pandas.core.sorting import (
compress_group_index,
decons_obs_group_ids,
- get_flattened_list,
get_group_index,
get_group_index_sorter,
get_indexer_dict,
@@ -414,6 +415,7 @@ def _call_cython_op(
"last",
"first",
"sum",
+ "median",
]:
func(
out=result,
@@ -424,8 +426,9 @@ def _call_cython_op(
mask=mask,
result_mask=result_mask,
is_datetimelike=is_datetimelike,
+ **kwargs,
)
- elif self.how in ["sem", "std", "var", "ohlc", "prod", "median"]:
+ elif self.how in ["sem", "std", "var", "ohlc", "prod"]:
if self.how in ["std", "sem"]:
kwargs["is_datetimelike"] = is_datetimelike
func(
@@ -603,9 +606,7 @@ def __iter__(self) -> Iterator[Hashable]:
def nkeys(self) -> int:
return len(self.groupings)
- def get_iterator(
- self, data: NDFrameT, axis: AxisInt = 0
- ) -> Iterator[tuple[Hashable, NDFrameT]]:
+ def get_iterator(self, data: NDFrameT) -> Iterator[tuple[Hashable, NDFrameT]]:
"""
Groupby iterator
@@ -614,38 +615,27 @@ def get_iterator(
Generator yielding sequence of (name, subsetted object)
for each group
"""
- splitter = self._get_splitter(data, axis=axis)
- keys = self.group_keys_seq
+ splitter = self._get_splitter(data)
+ # TODO: Would be more efficient to skip unobserved for transforms
+ keys = self.result_index
yield from zip(keys, splitter)
@final
- def _get_splitter(self, data: NDFrame, axis: AxisInt = 0) -> DataSplitter:
+ def _get_splitter(self, data: NDFrame) -> DataSplitter:
"""
Returns
-------
Generator yielding subsetted objects
"""
- ids, _, ngroups = self.group_info
+ ids, ngroups = self.group_info
return _get_splitter(
data,
ids,
ngroups,
sorted_ids=self._sorted_ids,
- sort_idx=self._sort_idx,
- axis=axis,
+ sort_idx=self.result_ilocs,
)
- @final
- @cache_readonly
- def group_keys_seq(self):
- if len(self.groupings) == 1:
- return self.levels[0]
- else:
- ids, _, ngroups = self.group_info
-
- # provide "flattened" iterator for multi-group setting
- return get_flattened_list(ids, ngroups, self.levels, self.codes)
-
@cache_readonly
def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
"""dict {group name -> group indices}"""
@@ -653,10 +643,10 @@ def indices(self) -> dict[Hashable, npt.NDArray[np.intp]]:
# This shows unused categories in indices GH#38642
return self.groupings[0].indices
codes_list = [ping.codes for ping in self.groupings]
- keys = [ping._group_index for ping in self.groupings]
- return get_indexer_dict(codes_list, keys)
+ return get_indexer_dict(codes_list, self.levels)
@final
+ @cache_readonly
def result_ilocs(self) -> npt.NDArray[np.intp]:
"""
Get the original integer locations of result_index in the input.
@@ -664,18 +654,15 @@ def result_ilocs(self) -> npt.NDArray[np.intp]:
# Original indices are where group_index would go via sorting.
# But when dropna is true, we need to remove null values while accounting for
# any gaps that then occur because of them.
- group_index = get_group_index(
- self.codes, self.shape, sort=self._sort, xnull=True
- )
- group_index, _ = compress_group_index(group_index, sort=self._sort)
+ ids = self.ids
if self.has_dropped_na:
- mask = np.where(group_index >= 0)
+ mask = np.where(ids >= 0)
# Count how many gaps are caused by previous null values for each position
- null_gaps = np.cumsum(group_index == -1)[mask]
- group_index = group_index[mask]
+ null_gaps = np.cumsum(ids == -1)[mask]
+ ids = ids[mask]
- result = get_group_index_sorter(group_index, self.ngroups)
+ result = get_group_index_sorter(ids, self.ngroups)
if self.has_dropped_na:
# Shift by the number of prior null gaps
@@ -683,14 +670,17 @@ def result_ilocs(self) -> npt.NDArray[np.intp]:
return result
- @final
@property
def codes(self) -> list[npt.NDArray[np.signedinteger]]:
return [ping.codes for ping in self.groupings]
@property
def levels(self) -> list[Index]:
- return [ping._group_index for ping in self.groupings]
+ if len(self.groupings) > 1:
+ # mypy doesn't know result_index must be a MultiIndex
+ return list(self.result_index.levels) # type: ignore[attr-defined]
+ else:
+ return [self.result_index]
@property
def names(self) -> list[Hashable]:
@@ -701,7 +691,7 @@ def size(self) -> Series:
"""
Compute group sizes.
"""
- ids, _, ngroups = self.group_info
+ ids, ngroups = self.group_info
out: np.ndarray | list
if ngroups:
out = np.bincount(ids[ids != -1], minlength=ngroups)
@@ -710,26 +700,25 @@ def size(self) -> Series:
return Series(out, index=self.result_index, dtype="int64", copy=False)
@cache_readonly
- def groups(self) -> dict[Hashable, np.ndarray]:
+ def groups(self) -> dict[Hashable, Index]:
"""dict {group name -> group labels}"""
if len(self.groupings) == 1:
return self.groupings[0].groups
- else:
- to_groupby = []
- for ping in self.groupings:
- gv = ping.grouping_vector
- if not isinstance(gv, BaseGrouper):
- to_groupby.append(gv)
- else:
- to_groupby.append(gv.groupings[0].grouping_vector)
- index = MultiIndex.from_arrays(to_groupby)
- return self.axis.groupby(index)
+ result_index, ids = self.result_index_and_ids
+ values = result_index._values
+ categories = Categorical(ids, categories=np.arange(len(result_index)))
+ result = {
+ # mypy is not aware that group has to be an integer
+ values[group]: self.axis.take(axis_ilocs) # type: ignore[call-overload]
+ for group, axis_ilocs in categories._reverse_indexer().items()
+ }
+ return result
@final
@cache_readonly
def is_monotonic(self) -> bool:
# return if my group orderings are monotonic
- return Index(self.group_info[0]).is_monotonic_increasing
+ return Index(self.ids).is_monotonic_increasing
@final
@cache_readonly
@@ -737,35 +726,18 @@ def has_dropped_na(self) -> bool:
"""
Whether grouper has null value(s) that are dropped.
"""
- return bool((self.group_info[0] < 0).any())
+ return bool((self.ids < 0).any())
@cache_readonly
- def group_info(self) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp], int]:
- comp_ids, obs_group_ids = self._get_compressed_codes()
-
- ngroups = len(obs_group_ids)
- comp_ids = ensure_platform_int(comp_ids)
-
- return comp_ids, obs_group_ids, ngroups
+ def group_info(self) -> tuple[npt.NDArray[np.intp], int]:
+ result_index, ids = self.result_index_and_ids
+ ngroups = len(result_index)
+ return ids, ngroups
@cache_readonly
def codes_info(self) -> npt.NDArray[np.intp]:
# return the codes of items in original grouped axis
- ids, _, _ = self.group_info
- return ids
-
- @final
- def _get_compressed_codes(
- self,
- ) -> tuple[npt.NDArray[np.signedinteger], npt.NDArray[np.intp]]:
- # The first returned ndarray may have any signed integer dtype
- if len(self.groupings) > 1:
- group_index = get_group_index(self.codes, self.shape, sort=True, xnull=True)
- return compress_group_index(group_index, sort=self._sort)
- # FIXME: compress_group_index's second return value is int64, not intp
-
- ping = self.groupings[0]
- return ping.codes, np.arange(len(ping._group_index), dtype=np.intp)
+ return self.ids
@final
@cache_readonly
@@ -773,37 +745,128 @@ def ngroups(self) -> int:
return len(self.result_index)
@property
- def reconstructed_codes(self) -> list[npt.NDArray[np.intp]]:
- codes = self.codes
- ids, obs_ids, _ = self.group_info
- return decons_obs_group_ids(ids, obs_ids, self.shape, codes, xnull=True)
+ def result_index(self) -> Index:
+ return self.result_index_and_ids[0]
+
+ @property
+ def ids(self) -> npt.NDArray[np.intp]:
+ return self.result_index_and_ids[1]
@cache_readonly
- def result_index(self) -> Index:
+ def result_index_and_ids(self) -> tuple[Index, npt.NDArray[np.intp]]:
+ levels = [Index._with_infer(ping.uniques) for ping in self.groupings]
+ obs = [
+ ping._observed or not ping._passed_categorical for ping in self.groupings
+ ]
+ # When passed a categorical grouping, keep all categories
+ for k, (ping, level) in enumerate(zip(self.groupings, levels)):
+ if ping._passed_categorical:
+ levels[k] = level.set_categories(ping._orig_cats)
+
if len(self.groupings) == 1:
- return self.groupings[0]._result_index.rename(self.names[0])
+ result_index = levels[0]
+ result_index.name = self.names[0]
+ ids = ensure_platform_int(self.codes[0])
+ elif all(obs):
+ result_index, ids = self._ob_index_and_ids(levels, self.codes, self.names)
+ elif not any(obs):
+ result_index, ids = self._unob_index_and_ids(levels, self.codes, self.names)
+ else:
+ # Combine unobserved and observed parts
+ names = self.names
+ codes = [ping.codes for ping in self.groupings]
+ ob_indices = [idx for idx, ob in enumerate(obs) if ob]
+ unob_indices = [idx for idx, ob in enumerate(obs) if not ob]
+ ob_index, ob_ids = self._ob_index_and_ids(
+ levels=[levels[idx] for idx in ob_indices],
+ codes=[codes[idx] for idx in ob_indices],
+ names=[names[idx] for idx in ob_indices],
+ )
+ unob_index, unob_ids = self._unob_index_and_ids(
+ levels=[levels[idx] for idx in unob_indices],
+ codes=[codes[idx] for idx in unob_indices],
+ names=[names[idx] for idx in unob_indices],
+ )
- codes = self.reconstructed_codes
- levels = [ping._result_index for ping in self.groupings]
- return MultiIndex(
- levels=levels, codes=codes, verify_integrity=False, names=self.names
+ result_index_codes = np.concatenate(
+ [
+ np.tile(unob_index.codes, len(ob_index)),
+ np.repeat(ob_index.codes, len(unob_index), axis=1),
+ ],
+ axis=0,
+ )
+ _, index = np.unique(unob_indices + ob_indices, return_index=True)
+ result_index = MultiIndex(
+ levels=list(unob_index.levels) + list(ob_index.levels),
+ codes=result_index_codes,
+ names=list(unob_index.names) + list(ob_index.names),
+ ).reorder_levels(index)
+ ids = len(unob_index) * ob_ids + unob_ids
+
+ if self._sort:
+ # Sort result_index and recode ids using the new order
+ sorter = result_index.argsort()
+ result_index = result_index.take(sorter)
+ _, index = np.unique(sorter, return_index=True)
+ ids = ensure_platform_int(ids)
+ ids = index.take(ids)
+ else:
+ # Recode ids and reorder result_index with observed groups up front,
+ # unobserved at the end
+ ids, uniques = compress_group_index(ids, sort=False)
+ ids = ensure_platform_int(ids)
+ taker = np.concatenate(
+ [uniques, np.delete(np.arange(len(result_index)), uniques)]
+ )
+ result_index = result_index.take(taker)
+
+ return result_index, ids
+
+ def _ob_index_and_ids(
+ self,
+ levels: list[Index],
+ codes: list[npt.NDArray[np.intp]],
+ names: list[Hashable],
+ ) -> tuple[MultiIndex, npt.NDArray[np.intp]]:
+ shape = tuple(len(level) for level in levels)
+ group_index = get_group_index(codes, shape, sort=True, xnull=True)
+ ob_ids, obs_group_ids = compress_group_index(group_index, sort=self._sort)
+ ob_ids = ensure_platform_int(ob_ids)
+ ob_index_codes = decons_obs_group_ids(
+ ob_ids, obs_group_ids, shape, codes, xnull=True
+ )
+ ob_index = MultiIndex(
+ levels=levels,
+ codes=ob_index_codes,
+ names=names,
+ verify_integrity=False,
)
+ ob_ids = ensure_platform_int(ob_ids)
+ return ob_index, ob_ids
+
+ def _unob_index_and_ids(
+ self,
+ levels: list[Index],
+ codes: list[npt.NDArray[np.intp]],
+ names: list[Hashable],
+ ) -> tuple[MultiIndex, npt.NDArray[np.intp]]:
+ shape = tuple(len(level) for level in levels)
+ unob_ids = get_group_index(codes, shape, sort=True, xnull=True)
+ unob_index = MultiIndex.from_product(levels, names=names)
+ unob_ids = ensure_platform_int(unob_ids)
+ return unob_index, unob_ids
@final
- def get_group_levels(self) -> list[ArrayLike]:
+ def get_group_levels(self) -> list[Index]:
# Note: only called from _insert_inaxis_grouper, which
# is only called for BaseGrouper, never for BinGrouper
+ result_index = self.result_index
if len(self.groupings) == 1:
- return [self.groupings[0]._group_arraylike]
-
- name_list = []
- for ping, codes in zip(self.groupings, self.reconstructed_codes):
- codes = ensure_platform_int(codes)
- levels = ping._group_arraylike.take(codes)
-
- name_list.append(levels)
-
- return name_list
+ return [result_index]
+ return [
+ result_index.get_level_values(level)
+ for level in range(result_index.nlevels)
+ ]
# ------------------------------------------------------------
# Aggregation functions
@@ -825,14 +888,12 @@ def _cython_operation(
cy_op = WrappedCythonOp(kind=kind, how=how, has_dropped_na=self.has_dropped_na)
- ids, _, _ = self.group_info
- ngroups = self.ngroups
return cy_op.cython_operation(
values=values,
axis=axis,
min_count=min_count,
- comp_ids=ids,
- ngroups=ngroups,
+ comp_ids=self.ids,
+ ngroups=self.ngroups,
**kwargs,
)
@@ -873,12 +934,10 @@ def agg_series(
def _aggregate_series_pure_python(
self, obj: Series, func: Callable
) -> npt.NDArray[np.object_]:
- _, _, ngroups = self.group_info
-
- result = np.empty(ngroups, dtype="O")
+ result = np.empty(self.ngroups, dtype="O")
initialized = False
- splitter = self._get_splitter(obj, axis=0)
+ splitter = self._get_splitter(obj)
for i, group in enumerate(splitter):
res = func(group)
@@ -895,11 +954,11 @@ def _aggregate_series_pure_python(
@final
def apply_groupwise(
- self, f: Callable, data: DataFrame | Series, axis: AxisInt = 0
+ self, f: Callable, data: DataFrame | Series
) -> tuple[list, bool]:
mutated = False
- splitter = self._get_splitter(data, axis=axis)
- group_keys = self.group_keys_seq
+ splitter = self._get_splitter(data)
+ group_keys = self.result_index
result_values = []
# This calls DataSplitter.__iter__
@@ -916,7 +975,7 @@ def apply_groupwise(
# group might be modified
group_axes = group.axes
res = f(group)
- if not mutated and not _is_indexed_like(res, group_axes, axis):
+ if not mutated and not _is_indexed_like(res, group_axes):
mutated = True
result_values.append(res)
# getattr pattern for __name__ is needed for functools.partial objects
@@ -935,18 +994,14 @@ def apply_groupwise(
# ------------------------------------------------------------
# Methods for sorting subsets of our GroupBy's object
- @final
- @cache_readonly
- def _sort_idx(self) -> npt.NDArray[np.intp]:
- # Counting sort indexer
- ids, _, ngroups = self.group_info
- return get_group_index_sorter(ids, ngroups)
-
@final
@cache_readonly
def _sorted_ids(self) -> npt.NDArray[np.intp]:
- ids, _, _ = self.group_info
- return ids.take(self._sort_idx)
+ result = self.ids.take(self.result_ilocs)
+ if getattr(self, "dropna", True):
+ # BinGrouper has no dropna
+ result = result[result >= 0]
+ return result
class BinGrouper(BaseGrouper):
@@ -1017,13 +1072,13 @@ def nkeys(self) -> int:
@cache_readonly
def codes_info(self) -> npt.NDArray[np.intp]:
# return the codes of items in original grouped axis
- ids, _, _ = self.group_info
+ ids = self.ids
if self.indexer is not None:
sorter = np.lexsort((ids, self.indexer))
ids = ids[sorter]
return ids
- def get_iterator(self, data: NDFrame, axis: AxisInt = 0):
+ def get_iterator(self, data: NDFrame):
"""
Groupby iterator
@@ -1032,12 +1087,7 @@ def get_iterator(self, data: NDFrame, axis: AxisInt = 0):
Generator yielding sequence of (name, subsetted object)
for each group
"""
- if axis == 0:
- slicer = lambda start, edge: data.iloc[start:edge]
- else:
- slicer = lambda start, edge: data.iloc[:, start:edge]
-
- length = len(data.axes[axis])
+ slicer = lambda start, edge: data.iloc[start:edge]
start = 0
for edge, label in zip(self.bins, self.binlabels):
@@ -1045,7 +1095,7 @@ def get_iterator(self, data: NDFrame, axis: AxisInt = 0):
yield label, slicer(start, edge)
start = edge
- if start < length:
+ if start < len(data):
yield self.binlabels[-1], slicer(start, None)
@cache_readonly
@@ -1061,34 +1111,30 @@ def indices(self):
return indices
@cache_readonly
- def group_info(self) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp], int]:
- ngroups = self.ngroups
- obs_group_ids = np.arange(ngroups, dtype=np.intp)
- rep = np.diff(np.r_[0, self.bins])
-
- rep = ensure_platform_int(rep)
- if ngroups == len(self.bins):
- comp_ids = np.repeat(np.arange(ngroups), rep)
- else:
- comp_ids = np.repeat(np.r_[-1, np.arange(ngroups)], rep)
-
- return (
- ensure_platform_int(comp_ids),
- obs_group_ids,
- ngroups,
- )
+ def group_info(self) -> tuple[npt.NDArray[np.intp], int]:
+ return self.ids, self.ngroups
@cache_readonly
- def reconstructed_codes(self) -> list[np.ndarray]:
- # get unique result indices, and prepend 0 as groupby starts from the first
- return [np.r_[0, np.flatnonzero(self.bins[1:] != self.bins[:-1]) + 1]]
+ def codes(self) -> list[npt.NDArray[np.intp]]:
+ return [self.ids]
@cache_readonly
- def result_index(self) -> Index:
+ def result_index_and_ids(self):
+ result_index = self.binlabels
if len(self.binlabels) != 0 and isna(self.binlabels[0]):
- return self.binlabels[1:]
+ result_index = result_index[1:]
- return self.binlabels
+ ngroups = len(result_index)
+ rep = np.diff(np.r_[0, self.bins])
+
+ rep = ensure_platform_int(rep)
+ if ngroups == len(self.bins):
+ ids = np.repeat(np.arange(ngroups), rep)
+ else:
+ ids = np.repeat(np.r_[-1, np.arange(ngroups)], rep)
+ ids = ensure_platform_int(ids)
+
+ return result_index, ids
@property
def levels(self) -> list[Index]:
@@ -1101,7 +1147,7 @@ def names(self) -> list[Hashable]:
@property
def groupings(self) -> list[grouper.Grouping]:
lev = self.binlabels
- codes = self.group_info[0]
+ codes = self.ids
labels = lev.take(codes)
ping = grouper.Grouping(
labels, labels, in_axis=False, level=None, uniques=lev._values
@@ -1109,13 +1155,13 @@ def groupings(self) -> list[grouper.Grouping]:
return [ping]
-def _is_indexed_like(obj, axes, axis: AxisInt) -> bool:
+def _is_indexed_like(obj, axes) -> bool:
if isinstance(obj, Series):
if len(axes) > 1:
return False
- return obj.axes[axis].equals(axes[axis])
+ return obj.index.equals(axes[0])
elif isinstance(obj, DataFrame):
- return obj.axes[axis].equals(axes[axis])
+ return obj.index.equals(axes[0])
return False
@@ -1133,7 +1179,6 @@ def __init__(
*,
sort_idx: npt.NDArray[np.intp],
sorted_ids: npt.NDArray[np.intp],
- axis: AxisInt = 0,
) -> None:
self.data = data
self.labels = ensure_platform_int(labels) # _should_ already be np.intp
@@ -1142,9 +1187,6 @@ def __init__(
self._slabels = sorted_ids
self._sort_idx = sort_idx
- self.axis = axis
- assert isinstance(axis, int), axis
-
def __iter__(self) -> Iterator:
sdata = self._sorted_data
@@ -1160,7 +1202,7 @@ def __iter__(self) -> Iterator:
@cache_readonly
def _sorted_data(self) -> NDFrameT:
- return self.data.take(self._sort_idx, axis=self.axis)
+ return self.data.take(self._sort_idx, axis=0)
def _chop(self, sdata, slice_obj: slice) -> NDFrame:
raise AbstractMethodError(self)
@@ -1178,11 +1220,8 @@ def _chop(self, sdata: Series, slice_obj: slice) -> Series:
class FrameSplitter(DataSplitter):
def _chop(self, sdata: DataFrame, slice_obj: slice) -> DataFrame:
# Fastpath equivalent to:
- # if self.axis == 0:
- # return sdata.iloc[slice_obj]
- # else:
- # return sdata.iloc[:, slice_obj]
- mgr = sdata._mgr.get_slice(slice_obj, axis=1 - self.axis)
+ # return sdata.iloc[slice_obj]
+ mgr = sdata._mgr.get_slice(slice_obj, axis=1)
df = sdata._constructor_from_mgr(mgr, axes=mgr.axes)
return df.__finalize__(sdata, method="groupby")
@@ -1194,7 +1233,6 @@ def _get_splitter(
*,
sort_idx: npt.NDArray[np.intp],
sorted_ids: npt.NDArray[np.intp],
- axis: AxisInt = 0,
) -> DataSplitter:
if isinstance(data, Series):
klass: type[DataSplitter] = SeriesSplitter
@@ -1202,6 +1240,4 @@ def _get_splitter(
# i.e. DataFrame
klass = FrameSplitter
- return klass(
- data, labels, ngroups, sort_idx=sort_idx, sorted_ids=sorted_ids, axis=axis
- )
+ return klass(data, labels, ngroups, sort_idx=sort_idx, sorted_ids=sorted_ids)
diff --git a/pandas/core/indexers/objects.py b/pandas/core/indexers/objects.py
index f2db4886a5590..2e6bcda520aba 100644
--- a/pandas/core/indexers/objects.py
+++ b/pandas/core/indexers/objects.py
@@ -1,4 +1,5 @@
"""Indexer objects for computing start/end window bounds for rolling operations"""
+
from __future__ import annotations
from datetime import timedelta
@@ -300,7 +301,7 @@ class FixedForwardWindowIndexer(BaseIndexer):
Examples
--------
- >>> df = pd.DataFrame({'B': [0, 1, 2, np.nan, 4]})
+ >>> df = pd.DataFrame({"B": [0, 1, 2, np.nan, 4]})
>>> df
B
0 0.0
@@ -399,7 +400,7 @@ def get_window_bounds(
start_arrays = []
end_arrays = []
window_indices_start = 0
- for key, indices in self.groupby_indices.items():
+ for indices in self.groupby_indices.values():
index_array: np.ndarray | None
if self.index_array is not None:
diff --git a/pandas/core/indexers/utils.py b/pandas/core/indexers/utils.py
index 55bb58f3108c3..b089be3469d87 100644
--- a/pandas/core/indexers/utils.py
+++ b/pandas/core/indexers/utils.py
@@ -1,6 +1,7 @@
"""
Low-dependency indexing utilities.
"""
+
from __future__ import annotations
from typing import (
@@ -202,7 +203,7 @@ def validate_indices(indices: np.ndarray, n: int) -> None:
Examples
--------
- >>> validate_indices(np.array([1, 2]), 3) # OK
+ >>> validate_indices(np.array([1, 2]), 3) # OK
>>> validate_indices(np.array([1, -2]), 3)
Traceback (most recent call last):
@@ -214,7 +215,7 @@ def validate_indices(indices: np.ndarray, n: int) -> None:
...
IndexError: indices are out-of-bounds
- >>> validate_indices(np.array([-1, -1]), 0) # OK
+ >>> validate_indices(np.array([-1, -1]), 0) # OK
>>> validate_indices(np.array([0, 1]), 0)
Traceback (most recent call last):
@@ -502,7 +503,7 @@ def check_array_indexer(array: AnyArrayLike, indexer: Any) -> Any:
For non-integer/boolean dtypes, an appropriate error is raised:
- >>> indexer = np.array([0., 2.], dtype="float64")
+ >>> indexer = np.array([0.0, 2.0], dtype="float64")
>>> pd.api.indexers.check_array_indexer(arr, indexer)
Traceback (most recent call last):
...
diff --git a/pandas/core/indexes/accessors.py b/pandas/core/indexes/accessors.py
index 7e3ba4089ff60..2bb234e174563 100644
--- a/pandas/core/indexes/accessors.py
+++ b/pandas/core/indexes/accessors.py
@@ -1,18 +1,18 @@
"""
datetimelike delegation
"""
+
from __future__ import annotations
from typing import (
TYPE_CHECKING,
+ NoReturn,
cast,
)
-import warnings
import numpy as np
from pandas._libs import lib
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.common import (
is_integer_dtype,
@@ -106,18 +106,9 @@ def _delegate_property_get(self, name: str):
else:
index = self._parent.index
# return the result as a Series
- result = Series(result, index=index, name=self.name).__finalize__(self._parent)
-
- # setting this object will show a SettingWithCopyWarning/Error
- result._is_copy = (
- "modifications to a property of a datetimelike "
- "object are not supported and are discarded. "
- "Change values on the original."
- )
-
- return result
+ return Series(result, index=index, name=self.name).__finalize__(self._parent)
- def _delegate_property_set(self, name: str, value, *args, **kwargs):
+ def _delegate_property_set(self, name: str, value, *args, **kwargs) -> NoReturn:
raise ValueError(
"modifications to a property of a datetimelike object are not supported. "
"Change values on the original."
@@ -134,19 +125,10 @@ def _delegate_method(self, name: str, *args, **kwargs):
if not is_list_like(result):
return result
- result = Series(result, index=self._parent.index, name=self.name).__finalize__(
+ return Series(result, index=self._parent.index, name=self.name).__finalize__(
self._parent
)
- # setting this object will show a SettingWithCopyWarning/Error
- result._is_copy = (
- "modifications to a method of a datetimelike "
- "object are not supported and are discarded. "
- "Change values on the original."
- )
-
- return result
-
@delegate_names(
delegate=ArrowExtensionArray,
@@ -230,16 +212,8 @@ def _delegate_method(self, name: str, *args, **kwargs):
def to_pytimedelta(self):
return cast(ArrowExtensionArray, self._parent.array)._dt_to_pytimedelta()
- def to_pydatetime(self):
+ def to_pydatetime(self) -> Series:
# GH#20306
- warnings.warn(
- f"The behavior of {type(self).__name__}.to_pydatetime is deprecated, "
- "in a future version this will return a Series containing python "
- "datetime objects instead of an ndarray. To retain the old behavior, "
- "call `np.array` on the result",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
return cast(ArrowExtensionArray, self._parent.array)._dt_to_pydatetime()
def isocalendar(self) -> DataFrame:
@@ -335,15 +309,9 @@ class DatetimeProperties(Properties):
Raises TypeError if the Series does not contain datetimelike values.
"""
- def to_pydatetime(self) -> np.ndarray:
+ def to_pydatetime(self) -> Series:
"""
- Return the data as an array of :class:`datetime.datetime` objects.
-
- .. deprecated:: 2.1.0
-
- The current behavior of dt.to_pydatetime is deprecated.
- In a future version this will return a Series containing python
- datetime objects instead of a ndarray.
+ Return the data as a Series of :class:`datetime.datetime` objects.
Timezone information is retained if present.
@@ -363,38 +331,34 @@ def to_pydatetime(self) -> np.ndarray:
Examples
--------
- >>> s = pd.Series(pd.date_range('20180310', periods=2))
+ >>> s = pd.Series(pd.date_range("20180310", periods=2))
>>> s
0 2018-03-10
1 2018-03-11
dtype: datetime64[ns]
>>> s.dt.to_pydatetime()
- array([datetime.datetime(2018, 3, 10, 0, 0),
- datetime.datetime(2018, 3, 11, 0, 0)], dtype=object)
+ 0 2018-03-10 00:00:00
+ 1 2018-03-11 00:00:00
+ dtype: object
pandas' nanosecond precision is truncated to microseconds.
- >>> s = pd.Series(pd.date_range('20180310', periods=2, freq='ns'))
+ >>> s = pd.Series(pd.date_range("20180310", periods=2, freq="ns"))
>>> s
0 2018-03-10 00:00:00.000000000
1 2018-03-10 00:00:00.000000001
dtype: datetime64[ns]
>>> s.dt.to_pydatetime()
- array([datetime.datetime(2018, 3, 10, 0, 0),
- datetime.datetime(2018, 3, 10, 0, 0)], dtype=object)
+ 0 2018-03-10 00:00:00
+ 1 2018-03-10 00:00:00
+ dtype: object
"""
# GH#20306
- warnings.warn(
- f"The behavior of {type(self).__name__}.to_pydatetime is deprecated, "
- "in a future version this will return a Series containing python "
- "datetime objects instead of an ndarray. To retain the old behavior, "
- "call `np.array` on the result",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self._get_values().to_pydatetime()
+ from pandas import Series
+
+ return Series(self._get_values().to_pydatetime(), dtype=object)
@property
def freq(self):
@@ -501,7 +465,7 @@ def to_pytimedelta(self) -> np.ndarray:
return self._get_values().to_pytimedelta()
@property
- def components(self):
+ def components(self) -> DataFrame:
"""
Return a Dataframe of the components of the Timedeltas.
@@ -511,7 +475,7 @@ def components(self):
Examples
--------
- >>> s = pd.Series(pd.to_timedelta(np.arange(5), unit='s'))
+ >>> s = pd.Series(pd.to_timedelta(np.arange(5), unit="s"))
>>> s
0 0 days 00:00:00
1 0 days 00:00:01
@@ -608,6 +572,44 @@ class PeriodProperties(Properties):
class CombinedDatetimelikeProperties(
DatetimeProperties, TimedeltaProperties, PeriodProperties
):
+ """
+ Accessor object for Series values' datetime-like, timedelta and period properties.
+
+ See Also
+ --------
+ DatetimeIndex : Index of datetime64 data.
+
+ Examples
+ --------
+ >>> dates = pd.Series(
+ ... ["2024-01-01", "2024-01-15", "2024-02-5"], dtype="datetime64[ns]"
+ ... )
+ >>> dates.dt.day
+ 0 1
+ 1 15
+ 2 5
+ dtype: int32
+ >>> dates.dt.month
+ 0 1
+ 1 1
+ 2 2
+ dtype: int32
+
+ >>> dates = pd.Series(
+ ... ["2024-01-01", "2024-01-15", "2024-02-5"], dtype="datetime64[ns, UTC]"
+ ... )
+ >>> dates.dt.day
+ 0 1
+ 1 15
+ 2 5
+ dtype: int32
+ >>> dates.dt.month
+ 0 1
+ 1 1
+ 2 2
+ dtype: int32
+ """
+
def __new__(cls, data: Series): # pyright: ignore[reportInconsistentConstructor]
# CombinedDatetimelikeProperties isn't really instantiated. Instead
# we need to choose which parent (datetime or timedelta) is
diff --git a/pandas/core/indexes/api.py b/pandas/core/indexes/api.py
index 560285bd57a22..9b05eb42c6d6e 100644
--- a/pandas/core/indexes/api.py
+++ b/pandas/core/indexes/api.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import textwrap
from typing import (
TYPE_CHECKING,
cast,
@@ -23,6 +22,7 @@
ensure_index,
ensure_index_from_sequences,
get_unanimous_names,
+ maybe_sequence_to_range,
)
from pandas.core.indexes.category import CategoricalIndex
from pandas.core.indexes.datetimes import DatetimeIndex
@@ -34,16 +34,6 @@
if TYPE_CHECKING:
from pandas._typing import Axis
-_sort_msg = textwrap.dedent(
- """\
-Sorting because non-concatenation axis is not aligned. A future version
-of pandas will change to not sort by default.
-
-To accept the future behavior, pass 'sort=False'.
-
-To retain the current behavior and silence the warning, pass 'sort=True'.
-"""
-)
__all__ = [
@@ -66,6 +56,7 @@
"all_indexes_same",
"default_index",
"safe_sort_index",
+ "maybe_sequence_to_range",
]
@@ -74,7 +65,6 @@ def get_objs_combined_axis(
intersect: bool = False,
axis: Axis = 0,
sort: bool = True,
- copy: bool = False,
) -> Index:
"""
Extract combined index: return intersection or union (depending on the
@@ -92,15 +82,13 @@ def get_objs_combined_axis(
The axis to extract indexes from.
sort : bool, default True
Whether the result index should come out sorted or not.
- copy : bool, default False
- If True, return a copy of the combined index.
Returns
-------
Index
"""
obs_idxes = [obj._get_axis(axis) for obj in objs]
- return _get_combined_index(obs_idxes, intersect=intersect, sort=sort, copy=copy)
+ return _get_combined_index(obs_idxes, intersect=intersect, sort=sort)
def _get_distinct_objs(objs: list[Index]) -> list[Index]:
@@ -121,7 +109,6 @@ def _get_combined_index(
indexes: list[Index],
intersect: bool = False,
sort: bool = False,
- copy: bool = False,
) -> Index:
"""
Return the union or intersection of indexes.
@@ -135,8 +122,6 @@ def _get_combined_index(
calculate the union.
sort : bool, default False
Whether the result index should come out sorted or not.
- copy : bool, default False
- If True, return a copy of the combined index.
Returns
-------
@@ -158,10 +143,6 @@ def _get_combined_index(
if sort:
index = safe_sort_index(index)
- # GH 29879
- if copy:
- index = index.copy()
-
return index
@@ -295,6 +276,7 @@ def _find_common_index_dtype(inds):
raise TypeError("Cannot join tz-naive with tz-aware DatetimeIndex")
if len(dtis) == len(indexes):
+ sort = True
result = indexes[0]
elif len(dtis) > 1:
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 166d6946beacf..30cf6f0b866ee 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -20,11 +20,7 @@
import numpy as np
-from pandas._config import (
- get_option,
- using_copy_on_write,
- using_pyarrow_string_dtype,
-)
+from pandas._config import get_option
from pandas._libs import (
NaT,
@@ -55,6 +51,7 @@
F,
IgnoreRaise,
IndexLabel,
+ IndexT,
JoinHow,
Level,
NaPosition,
@@ -71,7 +68,6 @@
from pandas.util._decorators import (
Appender,
cache_readonly,
- deprecate_nonkeyword_arguments,
doc,
)
from pandas.util._exceptions import (
@@ -178,7 +174,6 @@
disallow_ndim_indexing,
is_valid_positional_slice,
)
-from pandas.core.indexes.frozen import FrozenList
from pandas.core.missing import clean_reindex_fill_method
from pandas.core.ops import get_op_result_name
from pandas.core.ops.invalid import make_invalid_op
@@ -362,7 +357,7 @@ class Index(IndexOpsMixin, PandasObject):
>>> pd.Index([1, 2, 3])
Index([1, 2, 3], dtype='int64')
- >>> pd.Index(list('abc'))
+ >>> pd.Index(list("abc"))
Index(['a', 'b', 'c'], dtype='object')
>>> pd.Index([1, 2, 3], dtype="uint8")
@@ -725,7 +720,7 @@ def _format_duplicate_message(self) -> DataFrame:
Examples
--------
- >>> idx = pd.Index(['a', 'b', 'a'])
+ >>> idx = pd.Index(["a", "b", "a"])
>>> idx._format_duplicate_message()
positions
label
@@ -812,7 +807,7 @@ def is_(self, other) -> bool:
Examples
--------
- >>> idx1 = pd.Index(['1', '2', '3'])
+ >>> idx1 = pd.Index(["1", "2", "3"])
>>> idx1.is_(idx1.view())
True
@@ -883,6 +878,8 @@ def _engine(
# error: Item "ExtensionArray" of "Union[ExtensionArray,
# ndarray[Any, Any]]" has no attribute "_ndarray" [union-attr]
target_values = self._data._ndarray # type: ignore[union-attr]
+ elif is_string_dtype(self.dtype) and not is_object_dtype(self.dtype):
+ return libindex.StringEngine(target_values)
# error: Argument 1 to "ExtensionEngine" has incompatible type
# "ndarray[Any, Any]"; expected "ExtensionArray"
@@ -912,7 +909,7 @@ def __len__(self) -> int:
"""
return len(self._data)
- def __array__(self, dtype=None) -> np.ndarray:
+ def __array__(self, dtype=None, copy=None) -> np.ndarray:
"""
The array interface, return my values.
"""
@@ -949,6 +946,9 @@ def __array_ufunc__(self, ufunc: np.ufunc, method: str_t, *inputs, **kwargs):
elif method == "reduce":
result = lib.item_from_zerodim(result)
return result
+ elif is_scalar(result):
+ # e.g. matmul
+ return result
if result.dtype == np.float16:
result = result.astype(np.float32)
@@ -956,7 +956,7 @@ def __array_ufunc__(self, ufunc: np.ufunc, method: str_t, *inputs, **kwargs):
return self.__array_wrap__(result)
@final
- def __array_wrap__(self, result, context=None):
+ def __array_wrap__(self, result, context=None, return_scalar=False):
"""
Gets called after a ufunc and other functions e.g. np.split.
"""
@@ -1001,7 +1001,7 @@ def ravel(self, order: str_t = "C") -> Self:
Examples
--------
- >>> s = pd.Series([1, 2, 3], index=['a', 'b', 'c'])
+ >>> s = pd.Series([1, 2, 3], index=["a", "b", "c"])
>>> s.index.ravel()
Index(['a', 'b', 'c'], dtype='object')
"""
@@ -1010,7 +1010,7 @@ def ravel(self, order: str_t = "C") -> Self:
def view(self, cls=None):
# we need to see if we are subclassing an
# index type here
- if cls is not None and not hasattr(cls, "_typ"):
+ if cls is not None:
dtype = cls
if isinstance(cls, str):
dtype = pandas_dtype(cls)
@@ -1027,16 +1027,6 @@ def view(self, cls=None):
result = self._data.view(cls)
else:
- if cls is not None:
- warnings.warn(
- # GH#55709
- f"Passing a type in {type(self).__name__}.view is deprecated "
- "and will raise in a future version. "
- "Call view without any argument to retain the old behavior.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
result = self._view()
if isinstance(result, Index):
result._id = self._id
@@ -1071,7 +1061,7 @@ def astype(self, dtype, copy: bool = True):
>>> idx = pd.Index([1, 2, 3])
>>> idx
Index([1, 2, 3], dtype='int64')
- >>> idx.astype('float')
+ >>> idx.astype("float")
Index([1.0, 2.0, 3.0], dtype='float64')
"""
if dtype is not None:
@@ -1107,9 +1097,7 @@ def astype(self, dtype, copy: bool = True):
result._references.add_index_reference(result)
return result
- _index_shared_docs[
- "take"
- ] = """
+ _index_shared_docs["take"] = """
Return a new %(klass)s of the values selected by the indices.
For internal compatibility with numpy arrays.
@@ -1159,6 +1147,9 @@ def take(
indices = ensure_platform_int(indices)
allow_fill = self._maybe_disallow_fill(allow_fill, fill_value, indices)
+ if indices.ndim == 1 and lib.is_range_indexer(indices, len(self)):
+ return self.copy()
+
# Note: we discard fill_value and use self._na_value, only relevant
# in the case where allow_fill is True and fill_value is not None
values = self._values
@@ -1196,9 +1187,7 @@ def _maybe_disallow_fill(self, allow_fill: bool, fill_value, indices) -> bool:
allow_fill = False
return allow_fill
- _index_shared_docs[
- "repeat"
- ] = """
+ _index_shared_docs["repeat"] = """
Repeat elements of a %(klass)s.
Returns a new %(klass)s where each element of the current %(klass)s
@@ -1275,7 +1264,7 @@ def copy(
Examples
--------
- >>> idx = pd.Index(['a', 'b', 'c'])
+ >>> idx = pd.Index(["a", "b", "c"])
>>> new_idx = idx.copy()
>>> idx is new_idx
False
@@ -1372,16 +1361,19 @@ def _format_attrs(self) -> list[tuple[str_t, str_t | int | bool | None]]:
return attrs
@final
- def _get_level_names(self) -> Hashable | Sequence[Hashable]:
+ def _get_level_names(self) -> range | Sequence[Hashable]:
"""
Return a name or list of names with None replaced by the level number.
"""
if self._is_multi:
- return [
- level if name is None else name for level, name in enumerate(self.names)
- ]
+ return maybe_sequence_to_range(
+ [
+ level if name is None else name
+ for level, name in enumerate(self.names)
+ ]
+ )
else:
- return 0 if self.name is None else self.name
+ return range(1) if self.name is None else [self.name]
@final
def _mpl_repr(self) -> np.ndarray:
@@ -1390,36 +1382,6 @@ def _mpl_repr(self) -> np.ndarray:
return cast(np.ndarray, self.values)
return self.astype(object, copy=False)._values
- def format(
- self,
- name: bool = False,
- formatter: Callable | None = None,
- na_rep: str_t = "NaN",
- ) -> list[str_t]:
- """
- Render a string representation of the Index.
- """
- warnings.warn(
- # GH#55413
- f"{type(self).__name__}.format is deprecated and will be removed "
- "in a future version. Convert using index.astype(str) or "
- "index.map(formatter) instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- header = []
- if name:
- header.append(
- pprint_thing(self.name, escape_chars=("\t", "\r", "\n"))
- if self.name is not None
- else ""
- )
-
- if formatter is not None:
- return header + list(self.map(formatter))
-
- return self._format_with_header(header=header, na_rep=na_rep)
-
_default_na_rep = "NaN"
@final
@@ -1567,7 +1529,7 @@ def to_series(self, index=None, name: Hashable | None = None) -> Series:
Examples
--------
- >>> idx = pd.Index(['Ant', 'Bear', 'Cow'], name='animal')
+ >>> idx = pd.Index(["Ant", "Bear", "Cow"], name="animal")
By default, the original index and original name is reused.
@@ -1588,7 +1550,7 @@ def to_series(self, index=None, name: Hashable | None = None) -> Series:
To override the name of the resulting column, specify ``name``:
- >>> idx.to_series(name='zoo')
+ >>> idx.to_series(name="zoo")
animal
Ant Ant
Bear Bear
@@ -1631,7 +1593,7 @@ def to_frame(
Examples
--------
- >>> idx = pd.Index(['Ant', 'Bear', 'Cow'], name='animal')
+ >>> idx = pd.Index(["Ant", "Bear", "Cow"], name="animal")
>>> idx.to_frame()
animal
animal
@@ -1649,7 +1611,7 @@ def to_frame(
To override the name of the resulting column, specify `name`:
- >>> idx.to_frame(index=False, name='zoo')
+ >>> idx.to_frame(index=False, name="zoo")
zoo
0 Ant
1 Bear
@@ -1658,8 +1620,11 @@ def to_frame(
from pandas import DataFrame
if name is lib.no_default:
- name = self._get_level_names()
- result = DataFrame({name: self}, copy=not using_copy_on_write())
+ result_name = self._get_level_names()
+ else:
+ result_name = Index([name]) # type: ignore[assignment]
+ result = DataFrame(self, copy=False)
+ result.columns = result_name
if index:
result.index = self
@@ -1675,7 +1640,7 @@ def name(self) -> Hashable:
Examples
--------
- >>> idx = pd.Index([1, 2, 3], name='x')
+ >>> idx = pd.Index([1, 2, 3], name="x")
>>> idx
Index([1, 2, 3], dtype='int64', name='x')
>>> idx.name
@@ -1762,8 +1727,8 @@ def _get_default_index_names(
return names
- def _get_names(self) -> FrozenList:
- return FrozenList((self.name,))
+ def _get_names(self) -> tuple[Hashable | None, ...]:
+ return (self.name,)
def _set_names(self, values, *, level=None) -> None:
"""
@@ -1795,16 +1760,13 @@ def _set_names(self, values, *, level=None) -> None:
names = property(fset=_set_names, fget=_get_names)
@overload
- def set_names(self, names, *, level=..., inplace: Literal[False] = ...) -> Self:
- ...
+ def set_names(self, names, *, level=..., inplace: Literal[False] = ...) -> Self: ...
@overload
- def set_names(self, names, *, level=..., inplace: Literal[True]) -> None:
- ...
+ def set_names(self, names, *, level=..., inplace: Literal[True]) -> None: ...
@overload
- def set_names(self, names, *, level=..., inplace: bool = ...) -> Self | None:
- ...
+ def set_names(self, names, *, level=..., inplace: bool = ...) -> Self | None: ...
def set_names(self, names, *, level=None, inplace: bool = False) -> Self | None:
"""
@@ -1844,33 +1806,32 @@ def set_names(self, names, *, level=None, inplace: bool = False) -> Self | None:
>>> idx = pd.Index([1, 2, 3, 4])
>>> idx
Index([1, 2, 3, 4], dtype='int64')
- >>> idx.set_names('quarter')
+ >>> idx.set_names("quarter")
Index([1, 2, 3, 4], dtype='int64', name='quarter')
- >>> idx = pd.MultiIndex.from_product([['python', 'cobra'],
- ... [2018, 2019]])
+ >>> idx = pd.MultiIndex.from_product([["python", "cobra"], [2018, 2019]])
>>> idx
MultiIndex([('python', 2018),
('python', 2019),
( 'cobra', 2018),
( 'cobra', 2019)],
)
- >>> idx = idx.set_names(['kind', 'year'])
- >>> idx.set_names('species', level=0)
+ >>> idx = idx.set_names(["kind", "year"])
+ >>> idx.set_names("species", level=0)
MultiIndex([('python', 2018),
('python', 2019),
( 'cobra', 2018),
( 'cobra', 2019)],
- names=['species', 'year'])
+ names=('species', 'year'))
When renaming levels with a dict, levels can not be passed.
- >>> idx.set_names({'kind': 'snake'})
+ >>> idx.set_names({"kind": "snake"})
MultiIndex([('python', 2018),
('python', 2019),
( 'cobra', 2018),
( 'cobra', 2019)],
- names=['snake', 'year'])
+ names=('snake', 'year'))
"""
if level is not None and not isinstance(self, ABCMultiIndex):
raise ValueError("Level must be None for non-MultiIndex")
@@ -1912,17 +1873,12 @@ def set_names(self, names, *, level=None, inplace: bool = False) -> Self | None:
return None
@overload
- def rename(self, name, *, inplace: Literal[False] = ...) -> Self:
- ...
+ def rename(self, name, *, inplace: Literal[False] = ...) -> Self: ...
@overload
- def rename(self, name, *, inplace: Literal[True]) -> None:
- ...
+ def rename(self, name, *, inplace: Literal[True]) -> None: ...
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self", "name"], name="rename"
- )
- def rename(self, name, inplace: bool = False) -> Self | None:
+ def rename(self, name, *, inplace: bool = False) -> Self | None:
"""
Alter Index or MultiIndex name.
@@ -1948,26 +1904,26 @@ def rename(self, name, inplace: bool = False) -> Self | None:
Examples
--------
- >>> idx = pd.Index(['A', 'C', 'A', 'B'], name='score')
- >>> idx.rename('grade')
+ >>> idx = pd.Index(["A", "C", "A", "B"], name="score")
+ >>> idx.rename("grade")
Index(['A', 'C', 'A', 'B'], dtype='object', name='grade')
- >>> idx = pd.MultiIndex.from_product([['python', 'cobra'],
- ... [2018, 2019]],
- ... names=['kind', 'year'])
+ >>> idx = pd.MultiIndex.from_product(
+ ... [["python", "cobra"], [2018, 2019]], names=["kind", "year"]
+ ... )
>>> idx
MultiIndex([('python', 2018),
('python', 2019),
( 'cobra', 2018),
( 'cobra', 2019)],
- names=['kind', 'year'])
- >>> idx.rename(['species', 'year'])
+ names=('kind', 'year'))
+ >>> idx.rename(["species", "year"])
MultiIndex([('python', 2018),
('python', 2019),
( 'cobra', 2018),
( 'cobra', 2019)],
- names=['species', 'year'])
- >>> idx.rename('species')
+ names=('species', 'year'))
+ >>> idx.rename("species")
Traceback (most recent call last):
TypeError: Must pass list-like as `names`.
"""
@@ -2023,7 +1979,7 @@ def sortlevel(
ascending: bool | list[bool] = True,
sort_remaining=None,
na_position: NaPosition = "first",
- ):
+ ) -> tuple[Self, np.ndarray]:
"""
For internal compatibility with the Index API.
@@ -2090,7 +2046,7 @@ def _get_level_values(self, level) -> Index:
Examples
--------
- >>> idx = pd.Index(list('abc'))
+ >>> idx = pd.Index(list("abc"))
>>> idx
Index(['a', 'b', 'c'], dtype='object')
@@ -2125,34 +2081,35 @@ def droplevel(self, level: IndexLabel = 0):
Examples
--------
>>> mi = pd.MultiIndex.from_arrays(
- ... [[1, 2], [3, 4], [5, 6]], names=['x', 'y', 'z'])
+ ... [[1, 2], [3, 4], [5, 6]], names=["x", "y", "z"]
+ ... )
>>> mi
MultiIndex([(1, 3, 5),
(2, 4, 6)],
- names=['x', 'y', 'z'])
+ names=('x', 'y', 'z'))
>>> mi.droplevel()
MultiIndex([(3, 5),
(4, 6)],
- names=['y', 'z'])
+ names=('y', 'z'))
>>> mi.droplevel(2)
MultiIndex([(1, 3),
(2, 4)],
- names=['x', 'y'])
+ names=('x', 'y'))
- >>> mi.droplevel('z')
+ >>> mi.droplevel("z")
MultiIndex([(1, 3),
(2, 4)],
- names=['x', 'y'])
+ names=('x', 'y'))
- >>> mi.droplevel(['x', 'y'])
+ >>> mi.droplevel(["x", "y"])
Index([5, 6], dtype='int64', name='z')
"""
if not isinstance(level, (tuple, list)):
level = [level]
- levnums = sorted(self._get_level_number(lev) for lev in level)[::-1]
+ levnums = sorted((self._get_level_number(lev) for lev in level), reverse=True)
return self._drop_level_numbers(levnums)
@@ -2333,13 +2290,13 @@ def is_unique(self) -> bool:
>>> idx.is_unique
True
- >>> idx = pd.Index(["Watermelon", "Orange", "Apple",
- ... "Watermelon"]).astype("category")
+ >>> idx = pd.Index(["Watermelon", "Orange", "Apple", "Watermelon"]).astype(
+ ... "category"
+ ... )
>>> idx.is_unique
False
- >>> idx = pd.Index(["Orange", "Apple",
- ... "Watermelon"]).astype("category")
+ >>> idx = pd.Index(["Orange", "Apple", "Watermelon"]).astype("category")
>>> idx.is_unique
True
"""
@@ -2370,377 +2327,18 @@ def has_duplicates(self) -> bool:
>>> idx.has_duplicates
False
- >>> idx = pd.Index(["Watermelon", "Orange", "Apple",
- ... "Watermelon"]).astype("category")
+ >>> idx = pd.Index(["Watermelon", "Orange", "Apple", "Watermelon"]).astype(
+ ... "category"
+ ... )
>>> idx.has_duplicates
True
- >>> idx = pd.Index(["Orange", "Apple",
- ... "Watermelon"]).astype("category")
+ >>> idx = pd.Index(["Orange", "Apple", "Watermelon"]).astype("category")
>>> idx.has_duplicates
False
"""
return not self.is_unique
- @final
- def is_boolean(self) -> bool:
- """
- Check if the Index only consists of booleans.
-
- .. deprecated:: 2.0.0
- Use `pandas.api.types.is_bool_dtype` instead.
-
- Returns
- -------
- bool
- Whether or not the Index only consists of booleans.
-
- See Also
- --------
- is_integer : Check if the Index only consists of integers (deprecated).
- is_floating : Check if the Index is a floating type (deprecated).
- is_numeric : Check if the Index only consists of numeric data (deprecated).
- is_object : Check if the Index is of the object dtype (deprecated).
- is_categorical : Check if the Index holds categorical data.
- is_interval : Check if the Index holds Interval objects (deprecated).
-
- Examples
- --------
- >>> idx = pd.Index([True, False, True])
- >>> idx.is_boolean() # doctest: +SKIP
- True
-
- >>> idx = pd.Index(["True", "False", "True"])
- >>> idx.is_boolean() # doctest: +SKIP
- False
-
- >>> idx = pd.Index([True, False, "True"])
- >>> idx.is_boolean() # doctest: +SKIP
- False
- """
- warnings.warn(
- f"{type(self).__name__}.is_boolean is deprecated. "
- "Use pandas.api.types.is_bool_type instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self.inferred_type in ["boolean"]
-
- @final
- def is_integer(self) -> bool:
- """
- Check if the Index only consists of integers.
-
- .. deprecated:: 2.0.0
- Use `pandas.api.types.is_integer_dtype` instead.
-
- Returns
- -------
- bool
- Whether or not the Index only consists of integers.
-
- See Also
- --------
- is_boolean : Check if the Index only consists of booleans (deprecated).
- is_floating : Check if the Index is a floating type (deprecated).
- is_numeric : Check if the Index only consists of numeric data (deprecated).
- is_object : Check if the Index is of the object dtype. (deprecated).
- is_categorical : Check if the Index holds categorical data (deprecated).
- is_interval : Check if the Index holds Interval objects (deprecated).
-
- Examples
- --------
- >>> idx = pd.Index([1, 2, 3, 4])
- >>> idx.is_integer() # doctest: +SKIP
- True
-
- >>> idx = pd.Index([1.0, 2.0, 3.0, 4.0])
- >>> idx.is_integer() # doctest: +SKIP
- False
-
- >>> idx = pd.Index(["Apple", "Mango", "Watermelon"])
- >>> idx.is_integer() # doctest: +SKIP
- False
- """
- warnings.warn(
- f"{type(self).__name__}.is_integer is deprecated. "
- "Use pandas.api.types.is_integer_dtype instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self.inferred_type in ["integer"]
-
- @final
- def is_floating(self) -> bool:
- """
- Check if the Index is a floating type.
-
- .. deprecated:: 2.0.0
- Use `pandas.api.types.is_float_dtype` instead
-
- The Index may consist of only floats, NaNs, or a mix of floats,
- integers, or NaNs.
-
- Returns
- -------
- bool
- Whether or not the Index only consists of only consists of floats, NaNs, or
- a mix of floats, integers, or NaNs.
-
- See Also
- --------
- is_boolean : Check if the Index only consists of booleans (deprecated).
- is_integer : Check if the Index only consists of integers (deprecated).
- is_numeric : Check if the Index only consists of numeric data (deprecated).
- is_object : Check if the Index is of the object dtype. (deprecated).
- is_categorical : Check if the Index holds categorical data (deprecated).
- is_interval : Check if the Index holds Interval objects (deprecated).
-
- Examples
- --------
- >>> idx = pd.Index([1.0, 2.0, 3.0, 4.0])
- >>> idx.is_floating() # doctest: +SKIP
- True
-
- >>> idx = pd.Index([1.0, 2.0, np.nan, 4.0])
- >>> idx.is_floating() # doctest: +SKIP
- True
-
- >>> idx = pd.Index([1, 2, 3, 4, np.nan])
- >>> idx.is_floating() # doctest: +SKIP
- True
-
- >>> idx = pd.Index([1, 2, 3, 4])
- >>> idx.is_floating() # doctest: +SKIP
- False
- """
- warnings.warn(
- f"{type(self).__name__}.is_floating is deprecated. "
- "Use pandas.api.types.is_float_dtype instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self.inferred_type in ["floating", "mixed-integer-float", "integer-na"]
-
- @final
- def is_numeric(self) -> bool:
- """
- Check if the Index only consists of numeric data.
-
- .. deprecated:: 2.0.0
- Use `pandas.api.types.is_numeric_dtype` instead.
-
- Returns
- -------
- bool
- Whether or not the Index only consists of numeric data.
-
- See Also
- --------
- is_boolean : Check if the Index only consists of booleans (deprecated).
- is_integer : Check if the Index only consists of integers (deprecated).
- is_floating : Check if the Index is a floating type (deprecated).
- is_object : Check if the Index is of the object dtype. (deprecated).
- is_categorical : Check if the Index holds categorical data (deprecated).
- is_interval : Check if the Index holds Interval objects (deprecated).
-
- Examples
- --------
- >>> idx = pd.Index([1.0, 2.0, 3.0, 4.0])
- >>> idx.is_numeric() # doctest: +SKIP
- True
-
- >>> idx = pd.Index([1, 2, 3, 4.0])
- >>> idx.is_numeric() # doctest: +SKIP
- True
-
- >>> idx = pd.Index([1, 2, 3, 4])
- >>> idx.is_numeric() # doctest: +SKIP
- True
-
- >>> idx = pd.Index([1, 2, 3, 4.0, np.nan])
- >>> idx.is_numeric() # doctest: +SKIP
- True
-
- >>> idx = pd.Index([1, 2, 3, 4.0, np.nan, "Apple"])
- >>> idx.is_numeric() # doctest: +SKIP
- False
- """
- warnings.warn(
- f"{type(self).__name__}.is_numeric is deprecated. "
- "Use pandas.api.types.is_any_real_numeric_dtype instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self.inferred_type in ["integer", "floating"]
-
- @final
- def is_object(self) -> bool:
- """
- Check if the Index is of the object dtype.
-
- .. deprecated:: 2.0.0
- Use `pandas.api.types.is_object_dtype` instead.
-
- Returns
- -------
- bool
- Whether or not the Index is of the object dtype.
-
- See Also
- --------
- is_boolean : Check if the Index only consists of booleans (deprecated).
- is_integer : Check if the Index only consists of integers (deprecated).
- is_floating : Check if the Index is a floating type (deprecated).
- is_numeric : Check if the Index only consists of numeric data (deprecated).
- is_categorical : Check if the Index holds categorical data (deprecated).
- is_interval : Check if the Index holds Interval objects (deprecated).
-
- Examples
- --------
- >>> idx = pd.Index(["Apple", "Mango", "Watermelon"])
- >>> idx.is_object() # doctest: +SKIP
- True
-
- >>> idx = pd.Index(["Apple", "Mango", 2.0])
- >>> idx.is_object() # doctest: +SKIP
- True
-
- >>> idx = pd.Index(["Watermelon", "Orange", "Apple",
- ... "Watermelon"]).astype("category")
- >>> idx.is_object() # doctest: +SKIP
- False
-
- >>> idx = pd.Index([1.0, 2.0, 3.0, 4.0])
- >>> idx.is_object() # doctest: +SKIP
- False
- """
- warnings.warn(
- f"{type(self).__name__}.is_object is deprecated."
- "Use pandas.api.types.is_object_dtype instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return is_object_dtype(self.dtype)
-
- @final
- def is_categorical(self) -> bool:
- """
- Check if the Index holds categorical data.
-
- .. deprecated:: 2.0.0
- Use `isinstance(index.dtype, pd.CategoricalDtype)` instead.
-
- Returns
- -------
- bool
- True if the Index is categorical.
-
- See Also
- --------
- CategoricalIndex : Index for categorical data.
- is_boolean : Check if the Index only consists of booleans (deprecated).
- is_integer : Check if the Index only consists of integers (deprecated).
- is_floating : Check if the Index is a floating type (deprecated).
- is_numeric : Check if the Index only consists of numeric data (deprecated).
- is_object : Check if the Index is of the object dtype. (deprecated).
- is_interval : Check if the Index holds Interval objects (deprecated).
-
- Examples
- --------
- >>> idx = pd.Index(["Watermelon", "Orange", "Apple",
- ... "Watermelon"]).astype("category")
- >>> idx.is_categorical() # doctest: +SKIP
- True
-
- >>> idx = pd.Index([1, 3, 5, 7])
- >>> idx.is_categorical() # doctest: +SKIP
- False
-
- >>> s = pd.Series(["Peter", "Victor", "Elisabeth", "Mar"])
- >>> s
- 0 Peter
- 1 Victor
- 2 Elisabeth
- 3 Mar
- dtype: object
- >>> s.index.is_categorical() # doctest: +SKIP
- False
- """
- warnings.warn(
- f"{type(self).__name__}.is_categorical is deprecated."
- "Use pandas.api.types.is_categorical_dtype instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- return self.inferred_type in ["categorical"]
-
- @final
- def is_interval(self) -> bool:
- """
- Check if the Index holds Interval objects.
-
- .. deprecated:: 2.0.0
- Use `isinstance(index.dtype, pd.IntervalDtype)` instead.
-
- Returns
- -------
- bool
- Whether or not the Index holds Interval objects.
-
- See Also
- --------
- IntervalIndex : Index for Interval objects.
- is_boolean : Check if the Index only consists of booleans (deprecated).
- is_integer : Check if the Index only consists of integers (deprecated).
- is_floating : Check if the Index is a floating type (deprecated).
- is_numeric : Check if the Index only consists of numeric data (deprecated).
- is_object : Check if the Index is of the object dtype. (deprecated).
- is_categorical : Check if the Index holds categorical data (deprecated).
-
- Examples
- --------
- >>> idx = pd.Index([pd.Interval(left=0, right=5),
- ... pd.Interval(left=5, right=10)])
- >>> idx.is_interval() # doctest: +SKIP
- True
-
- >>> idx = pd.Index([1, 3, 5, 7])
- >>> idx.is_interval() # doctest: +SKIP
- False
- """
- warnings.warn(
- f"{type(self).__name__}.is_interval is deprecated."
- "Use pandas.api.types.is_interval_dtype instead",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self.inferred_type in ["interval"]
-
- @final
- def _holds_integer(self) -> bool:
- """
- Whether the type is an integer type.
- """
- return self.inferred_type in ["integer", "mixed-integer"]
-
- @final
- def holds_integer(self) -> bool:
- """
- Whether the type is an integer type.
-
- .. deprecated:: 2.0.0
- Use `pandas.api.types.infer_dtype` instead
- """
- warnings.warn(
- f"{type(self).__name__}.holds_integer is deprecated. "
- "Use pandas.api.types.infer_dtype instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return self._holds_integer()
-
@cache_readonly
def inferred_type(self) -> str_t:
"""
@@ -2827,7 +2425,7 @@ def hasnans(self) -> bool:
Examples
--------
- >>> s = pd.Series([1, 2, 3], index=['a', 'b', None])
+ >>> s = pd.Series([1, 2, 3], index=["a", "b", None])
>>> s
a 1
b 2
@@ -2878,7 +2476,7 @@ def isna(self) -> npt.NDArray[np.bool_]:
Empty strings are not considered NA values. None is considered an NA
value.
- >>> idx = pd.Index(['black', '', 'red', None])
+ >>> idx = pd.Index(["black", "", "red", None])
>>> idx
Index(['black', '', 'red', None], dtype='object')
>>> idx.isna()
@@ -2886,8 +2484,9 @@ def isna(self) -> npt.NDArray[np.bool_]:
For datetimes, `NaT` (Not a Time) is considered as an NA value.
- >>> idx = pd.DatetimeIndex([pd.Timestamp('1940-04-25'),
- ... pd.Timestamp(''), None, pd.NaT])
+ >>> idx = pd.DatetimeIndex(
+ ... [pd.Timestamp("1940-04-25"), pd.Timestamp(""), None, pd.NaT]
+ ... )
>>> idx
DatetimeIndex(['1940-04-25', 'NaT', 'NaT', 'NaT'],
dtype='datetime64[ns]', freq=None)
@@ -2934,7 +2533,7 @@ def notna(self) -> npt.NDArray[np.bool_]:
Empty strings are not considered NA values. None is considered a NA
value.
- >>> idx = pd.Index(['black', '', 'red', None])
+ >>> idx = pd.Index(["black", "", "red", None])
>>> idx
Index(['black', '', 'red', None], dtype='object')
>>> idx.notna()
@@ -2944,7 +2543,7 @@ def notna(self) -> npt.NDArray[np.bool_]:
notnull = notna
- def fillna(self, value=None, downcast=lib.no_default):
+ def fillna(self, value=None):
"""
Fill NA/NaN values with the specified value.
@@ -2953,12 +2552,6 @@ def fillna(self, value=None, downcast=lib.no_default):
value : scalar
Scalar value to use to fill holes (e.g. 0).
This value cannot be a list-likes.
- downcast : dict, default is None
- A dict of item->dtype of what to downcast if possible,
- or the string 'infer' which will try to downcast to an appropriate
- equal type (e.g. float64 to int64 if possible).
-
- .. deprecated:: 2.1.0
Returns
-------
@@ -2977,28 +2570,13 @@ def fillna(self, value=None, downcast=lib.no_default):
"""
if not is_scalar(value):
raise TypeError(f"'value' must be a scalar, passed: {type(value).__name__}")
- if downcast is not lib.no_default:
- warnings.warn(
- f"The 'downcast' keyword in {type(self).__name__}.fillna is "
- "deprecated and will be removed in a future version. "
- "It was previously silently ignored.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- downcast = None
if self.hasnans:
result = self.putmask(self._isnan, value)
- if downcast is None:
- # no need to care metadata other than name
- # because it can't have freq if it has NaTs
- # _with_infer needed for test_fillna_categorical
- return Index._with_infer(result, name=self.name)
- raise NotImplementedError(
- f"{type(self).__name__}.fillna does not support 'downcast' "
- "argument values other than 'None'."
- )
+ # no need to care metadata other than name
+ # because it can't have freq if it has NaTs
+ # _with_infer needed for test_fillna_categorical
+ return Index._with_infer(result, name=self.name)
return self._view()
def dropna(self, how: AnyAll = "any") -> Self:
@@ -3094,20 +2672,20 @@ def drop_duplicates(self, *, keep: DropKeep = "first") -> Self:
--------
Generate an pandas.Index with duplicate values.
- >>> idx = pd.Index(['lama', 'cow', 'lama', 'beetle', 'lama', 'hippo'])
+ >>> idx = pd.Index(["llama", "cow", "llama", "beetle", "llama", "hippo"])
The `keep` parameter controls which duplicate values are removed.
The value 'first' keeps the first occurrence for each
set of duplicated entries. The default value of keep is 'first'.
- >>> idx.drop_duplicates(keep='first')
- Index(['lama', 'cow', 'beetle', 'hippo'], dtype='object')
+ >>> idx.drop_duplicates(keep="first")
+ Index(['llama', 'cow', 'beetle', 'hippo'], dtype='object')
The value 'last' keeps the last occurrence for each set of duplicated
entries.
- >>> idx.drop_duplicates(keep='last')
- Index(['cow', 'beetle', 'lama', 'hippo'], dtype='object')
+ >>> idx.drop_duplicates(keep="last")
+ Index(['cow', 'beetle', 'llama', 'hippo'], dtype='object')
The value ``False`` discards all sets of duplicated entries.
@@ -3153,19 +2731,19 @@ def duplicated(self, keep: DropKeep = "first") -> npt.NDArray[np.bool_]:
By default, for each set of duplicated values, the first occurrence is
set to False and all others to True:
- >>> idx = pd.Index(['lama', 'cow', 'lama', 'beetle', 'lama'])
+ >>> idx = pd.Index(["llama", "cow", "llama", "beetle", "llama"])
>>> idx.duplicated()
array([False, False, True, False, True])
which is equivalent to
- >>> idx.duplicated(keep='first')
+ >>> idx.duplicated(keep="first")
array([False, False, True, False, True])
By using 'last', the last occurrence of each set of duplicated values
is set on False and all others on True:
- >>> idx.duplicated(keep='last')
+ >>> idx.duplicated(keep="last")
array([ True, False, True, False, False])
By setting keep on ``False``, all duplicates are True:
@@ -3274,7 +2852,7 @@ def union(self, other, sort=None):
Union mismatched dtypes
- >>> idx1 = pd.Index(['a', 'b', 'c', 'd'])
+ >>> idx1 = pd.Index(["a", "b", "c", "d"])
>>> idx2 = pd.Index([1, 2, 3, 4])
>>> idx1.union(idx2)
Index(['a', 'b', 'c', 'd', 1, 2, 3, 4], dtype='object')
@@ -3656,9 +3234,12 @@ def difference(self, other, sort=None):
def _difference(self, other, sort):
# overridden by RangeIndex
+ this = self
+ if isinstance(self, ABCCategoricalIndex) and self.hasnans and other.hasnans:
+ this = this.dropna()
other = other.unique()
- the_diff = self[other.get_indexer_for(self) == -1]
- the_diff = the_diff if self.is_unique else the_diff.unique()
+ the_diff = this[other.get_indexer_for(this) == -1]
+ the_diff = the_diff if this.is_unique else the_diff.unique()
the_diff = _maybe_try_sort(the_diff, sort)
return the_diff
@@ -3778,16 +3359,16 @@ def get_loc(self, key):
Examples
--------
- >>> unique_index = pd.Index(list('abc'))
- >>> unique_index.get_loc('b')
+ >>> unique_index = pd.Index(list("abc"))
+ >>> unique_index.get_loc("b")
1
- >>> monotonic_index = pd.Index(list('abbc'))
- >>> monotonic_index.get_loc('b')
+ >>> monotonic_index = pd.Index(list("abbc"))
+ >>> monotonic_index.get_loc("b")
slice(1, 3, None)
- >>> non_monotonic_index = pd.Index(list('abcb'))
- >>> non_monotonic_index.get_loc('b')
+ >>> non_monotonic_index = pd.Index(list("abcb"))
+ >>> non_monotonic_index.get_loc("b")
array([False, True, False, True])
"""
casted_key = self._maybe_cast_indexer(key)
@@ -3798,7 +3379,7 @@ def get_loc(self, key):
isinstance(casted_key, abc.Iterable)
and any(isinstance(x, slice) for x in casted_key)
):
- raise InvalidIndexError(key)
+ raise InvalidIndexError(key) from err
raise KeyError(key) from err
except TypeError:
# If we have a listlike key, _check_indexing_error will raise
@@ -3858,8 +3439,8 @@ def get_indexer(
Examples
--------
- >>> index = pd.Index(['c', 'a', 'b'])
- >>> index.get_indexer(['a', 'b', 'x'])
+ >>> index = pd.Index(["c", "a", "b"])
+ >>> index.get_indexer(["a", "b", "x"])
array([ 1, 2, -1])
Notice that the return value is an array of locations in ``index``
@@ -4043,7 +3624,7 @@ def _convert_tolerance(self, tolerance, target: np.ndarray | Index) -> np.ndarra
raise ValueError(
f"tolerance argument for {type(self).__name__} with dtype {self.dtype} "
- f"must be numeric if it is a scalar: {repr(tolerance)}"
+ f"must be numeric if it is a scalar: {tolerance!r}"
)
return tolerance
@@ -4090,7 +3671,7 @@ def _get_fill_indexer_searchsorted(
"""
if limit is not None:
raise ValueError(
- f"limit argument for {repr(method)} method only well-defined "
+ f"limit argument for {method!r} method only well-defined "
"if index and target are monotonic"
)
@@ -4369,10 +3950,10 @@ def reindex(
Examples
--------
- >>> idx = pd.Index(['car', 'bike', 'train', 'tractor'])
+ >>> idx = pd.Index(["car", "bike", "train", "tractor"])
>>> idx
Index(['car', 'bike', 'train', 'tractor'], dtype='object')
- >>> idx.reindex(['car', 'bike'])
+ >>> idx.reindex(["car", "bike"])
(Index(['car', 'bike'], dtype='object'), array([0, 1]))
"""
# GH6552: preserve names when reindexing to non-named target
@@ -4427,7 +4008,7 @@ def _wrap_reindex_result(self, target, indexer, preserve_names: bool):
target = self._maybe_preserve_names(target, preserve_names)
return target
- def _maybe_preserve_names(self, target: Index, preserve_names: bool):
+ def _maybe_preserve_names(self, target: IndexT, preserve_names: bool) -> IndexT:
if preserve_names and target.nlevels == 1 and target.name != self.name:
target = target.copy(deep=False)
target.name = self.name
@@ -4517,8 +4098,7 @@ def join(
level: Level = ...,
return_indexers: Literal[True],
sort: bool = ...,
- ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
- ...
+ ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]: ...
@overload
def join(
@@ -4529,8 +4109,7 @@ def join(
level: Level = ...,
return_indexers: Literal[False] = ...,
sort: bool = ...,
- ) -> Index:
- ...
+ ) -> Index: ...
@overload
def join(
@@ -4541,8 +4120,9 @@ def join(
level: Level = ...,
return_indexers: bool = ...,
sort: bool = ...,
- ) -> Index | tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
- ...
+ ) -> (
+ Index | tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]
+ ): ...
@final
@_maybe_return_indexers
@@ -4576,7 +4156,7 @@ def join(
--------
>>> idx1 = pd.Index([1, 2, 3])
>>> idx2 = pd.Index([4, 5, 6])
- >>> idx1.join(idx2, how='outer')
+ >>> idx1.join(idx2, how="outer")
Index([1, 2, 3, 4, 5, 6], dtype='int64')
"""
other = ensure_index(other)
@@ -4608,38 +4188,12 @@ def join(
if level is not None and (self._is_multi or other._is_multi):
return self._join_level(other, level, how=how)
- lidx: np.ndarray | None
- ridx: np.ndarray | None
-
- if len(other) == 0:
- if how in ("left", "outer"):
- if sort and not self.is_monotonic_increasing:
- lidx = self.argsort()
- join_index = self.take(lidx)
- else:
- lidx = None
- join_index = self._view()
- ridx = np.broadcast_to(np.intp(-1), len(join_index))
- return join_index, lidx, ridx
- elif how in ("right", "inner", "cross"):
- join_index = other._view()
- lidx = np.array([], dtype=np.intp)
- return join_index, lidx, None
-
- if len(self) == 0:
- if how in ("right", "outer"):
- if sort and not other.is_monotonic_increasing:
- ridx = other.argsort()
- join_index = other.take(ridx)
- else:
- ridx = None
- join_index = other._view()
- lidx = np.broadcast_to(np.intp(-1), len(join_index))
- return join_index, lidx, ridx
- elif how in ("left", "inner", "cross"):
- join_index = self._view()
- ridx = np.array([], dtype=np.intp)
- return join_index, None, ridx
+ if len(self) == 0 or len(other) == 0:
+ try:
+ return self._join_empty(other, how, sort)
+ except TypeError:
+ # object dtype; non-comparable objects
+ pass
if self.dtype != other.dtype:
dtype = self._find_common_type_compat(other)
@@ -4674,6 +4228,32 @@ def join(
return self._join_via_get_indexer(other, how, sort)
+ def _join_empty(
+ self, other: Index, how: JoinHow, sort: bool
+ ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
+ assert len(self) == 0 or len(other) == 0
+ _validate_join_method(how)
+
+ lidx: np.ndarray | None
+ ridx: np.ndarray | None
+
+ if len(other):
+ how = cast(JoinHow, {"left": "right", "right": "left"}.get(how, how))
+ join_index, ridx, lidx = other._join_empty(self, how, sort)
+ elif how in ["left", "outer"]:
+ if sort and not self.is_monotonic_increasing:
+ lidx = self.argsort()
+ join_index = self.take(lidx)
+ else:
+ lidx = None
+ join_index = self._view()
+ ridx = np.broadcast_to(np.intp(-1), len(join_index))
+ else:
+ join_index = other._view()
+ lidx = np.array([], dtype=np.intp)
+ ridx = None
+ return join_index, lidx, ridx
+
@final
def _join_via_get_indexer(
self, other: Index, how: JoinHow, sort: bool
@@ -4682,11 +4262,23 @@ def _join_via_get_indexer(
# uniqueness/monotonicity
# Note: at this point we have checked matching dtypes
+ lindexer: npt.NDArray[np.intp] | None
+ rindexer: npt.NDArray[np.intp] | None
if how == "left":
- join_index = self.sort_values() if sort else self
+ if sort:
+ join_index, lindexer = self.sort_values(return_indexer=True)
+ rindexer = other.get_indexer_for(join_index)
+ return join_index, lindexer, rindexer
+ else:
+ join_index = self
elif how == "right":
- join_index = other.sort_values() if sort else other
+ if sort:
+ join_index, rindexer = other.sort_values(return_indexer=True)
+ lindexer = self.get_indexer_for(join_index)
+ return join_index, lindexer, rindexer
+ else:
+ join_index = other
elif how == "inner":
join_index = self.intersection(other, sort=sort)
elif how == "outer":
@@ -4699,6 +4291,10 @@ def _join_via_get_indexer(
except TypeError:
pass
+ names = other.names if how == "right" else self.names
+ if join_index.names != names:
+ join_index = join_index.set_names(names)
+
if join_index is self:
lindexer = None
else:
@@ -4810,11 +4406,18 @@ def _join_non_unique(
left_idx, right_idx = get_join_indexers_non_unique(
self._values, other._values, how=how, sort=sort
)
- mask = left_idx == -1
- join_idx = self.take(left_idx)
- right = other.take(right_idx)
- join_index = join_idx.putmask(mask, right)
+ if how == "right":
+ join_index = other.take(right_idx)
+ else:
+ join_index = self.take(left_idx)
+
+ if how == "outer":
+ mask = left_idx == -1
+ if mask.any():
+ right = other.take(right_idx)
+ join_index = join_index.putmask(mask, right)
+
if isinstance(join_index, ABCMultiIndex) and how == "outer":
# test_join_index_levels
join_index = join_index._sort_levels_monotonic()
@@ -4835,7 +4438,9 @@ def _join_level(
"""
from pandas.core.indexes.multi import MultiIndex
- def _get_leaf_sorter(labels: list[np.ndarray]) -> npt.NDArray[np.intp]:
+ def _get_leaf_sorter(
+ labels: tuple[np.ndarray, ...] | list[np.ndarray],
+ ) -> npt.NDArray[np.intp]:
"""
Returns sorter for the inner most level while preserving the
order of higher levels.
@@ -4969,7 +4574,6 @@ def _get_leaf_sorter(labels: list[np.ndarray]) -> npt.NDArray[np.intp]:
)
return join_index, left_indexer, right_indexer
- @final
def _join_monotonic(
self, other: Index, how: JoinHow = "left"
) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
@@ -4990,62 +4594,71 @@ def _join_monotonic(
ridx: npt.NDArray[np.intp] | None
lidx: npt.NDArray[np.intp] | None
- if self.is_unique and other.is_unique:
- # We can perform much better than the general case
- if how == "left":
+ if how == "left":
+ if other.is_unique:
+ # We can perform much better than the general case
join_index = self
lidx = None
ridx = self._left_indexer_unique(other)
- elif how == "right":
+ else:
+ join_array, lidx, ridx = self._left_indexer(other)
+ join_index, lidx, ridx = self._wrap_join_result(
+ join_array, other, lidx, ridx, how
+ )
+ elif how == "right":
+ if self.is_unique:
+ # We can perform much better than the general case
join_index = other
lidx = other._left_indexer_unique(self)
ridx = None
- elif how == "inner":
- join_array, lidx, ridx = self._inner_indexer(other)
- join_index = self._wrap_joined_index(join_array, other, lidx, ridx)
- elif how == "outer":
- join_array, lidx, ridx = self._outer_indexer(other)
- join_index = self._wrap_joined_index(join_array, other, lidx, ridx)
- else:
- if how == "left":
- join_array, lidx, ridx = self._left_indexer(other)
- elif how == "right":
+ else:
join_array, ridx, lidx = other._left_indexer(self)
- elif how == "inner":
- join_array, lidx, ridx = self._inner_indexer(other)
- elif how == "outer":
- join_array, lidx, ridx = self._outer_indexer(other)
-
- assert lidx is not None
- assert ridx is not None
-
- join_index = self._wrap_joined_index(join_array, other, lidx, ridx)
+ join_index, lidx, ridx = self._wrap_join_result(
+ join_array, other, lidx, ridx, how
+ )
+ elif how == "inner":
+ join_array, lidx, ridx = self._inner_indexer(other)
+ join_index, lidx, ridx = self._wrap_join_result(
+ join_array, other, lidx, ridx, how
+ )
+ elif how == "outer":
+ join_array, lidx, ridx = self._outer_indexer(other)
+ join_index, lidx, ridx = self._wrap_join_result(
+ join_array, other, lidx, ridx, how
+ )
lidx = None if lidx is None else ensure_platform_int(lidx)
ridx = None if ridx is None else ensure_platform_int(ridx)
return join_index, lidx, ridx
- def _wrap_joined_index(
+ def _wrap_join_result(
self,
joined: ArrayLike,
other: Self,
- lidx: npt.NDArray[np.intp],
- ridx: npt.NDArray[np.intp],
- ) -> Self:
+ lidx: npt.NDArray[np.intp] | None,
+ ridx: npt.NDArray[np.intp] | None,
+ how: JoinHow,
+ ) -> tuple[Self, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
assert other.dtype == self.dtype
- if isinstance(self, ABCMultiIndex):
- name = self.names if self.names == other.names else None
- # error: Incompatible return value type (got "MultiIndex",
- # expected "Self")
- mask = lidx == -1
- join_idx = self.take(lidx)
- right = cast("MultiIndex", other.take(ridx))
- join_index = join_idx.putmask(mask, right)._sort_levels_monotonic()
- return join_index.set_names(name) # type: ignore[return-value]
+ if lidx is not None and lib.is_range_indexer(lidx, len(self)):
+ lidx = None
+ if ridx is not None and lib.is_range_indexer(ridx, len(other)):
+ ridx = None
+
+ # return self or other if possible to maintain cached attributes
+ if lidx is None:
+ join_index = self
+ elif ridx is None:
+ join_index = other
else:
- name = get_op_result_name(self, other)
- return self._constructor._with_infer(joined, name=name, dtype=self.dtype)
+ join_index = self._constructor._with_infer(joined, dtype=self.dtype)
+
+ names = other.names if how == "right" else self.names
+ if join_index.names != names:
+ join_index = join_index.set_names(names)
+
+ return join_index, lidx, ridx
@final
@cache_readonly
@@ -5119,13 +4732,11 @@ def values(self) -> ArrayLike:
[(0, 1], (1, 2], (2, 3], (3, 4], (4, 5]]
Length: 5, dtype: interval[int64, right]
"""
- if using_copy_on_write():
- data = self._data
- if isinstance(data, np.ndarray):
- data = data.view()
- data.flags.writeable = False
- return data
- return self._data
+ data = self._data
+ if isinstance(data, np.ndarray):
+ data = data.view()
+ data.flags.writeable = False
+ return data
@cache_readonly
@doc(IndexOpsMixin.array)
@@ -5263,10 +4874,10 @@ def where(self, cond, other=None) -> Index:
Examples
--------
- >>> idx = pd.Index(['car', 'bike', 'train', 'tractor'])
+ >>> idx = pd.Index(["car", "bike", "train", "tractor"])
>>> idx
Index(['car', 'bike', 'train', 'tractor'], dtype='object')
- >>> idx.where(idx.isin(['car', 'train']), 'other')
+ >>> idx.where(idx.isin(["car", "train"]), "other")
Index(['car', 'other', 'train', 'other'], dtype='object')
"""
if isinstance(self, ABCMultiIndex):
@@ -5595,10 +5206,10 @@ def equals(self, other: Any) -> bool:
The dtype is *not* compared
- >>> int64_idx = pd.Index([1, 2, 3], dtype='int64')
+ >>> int64_idx = pd.Index([1, 2, 3], dtype="int64")
>>> int64_idx
Index([1, 2, 3], dtype='int64')
- >>> uint64_idx = pd.Index([1, 2, 3], dtype='uint64')
+ >>> uint64_idx = pd.Index([1, 2, 3], dtype="uint64")
>>> uint64_idx
Index([1, 2, 3], dtype='uint64')
>>> int64_idx.equals(uint64_idx)
@@ -5657,13 +5268,13 @@ def identical(self, other) -> bool:
Examples
--------
- >>> idx1 = pd.Index(['1', '2', '3'])
- >>> idx2 = pd.Index(['1', '2', '3'])
+ >>> idx1 = pd.Index(["1", "2", "3"])
+ >>> idx2 = pd.Index(["1", "2", "3"])
>>> idx2.identical(idx1)
True
- >>> idx1 = pd.Index(['1', '2', '3'], name="A")
- >>> idx2 = pd.Index(['1', '2', '3'], name="B")
+ >>> idx1 = pd.Index(["1", "2", "3"], name="A")
+ >>> idx2 = pd.Index(["1", "2", "3"], name="B")
>>> idx2.identical(idx1)
False
"""
@@ -5711,39 +5322,38 @@ def asof(self, label):
--------
`Index.asof` returns the latest index label up to the passed label.
- >>> idx = pd.Index(['2013-12-31', '2014-01-02', '2014-01-03'])
- >>> idx.asof('2014-01-01')
+ >>> idx = pd.Index(["2013-12-31", "2014-01-02", "2014-01-03"])
+ >>> idx.asof("2014-01-01")
'2013-12-31'
If the label is in the index, the method returns the passed label.
- >>> idx.asof('2014-01-02')
+ >>> idx.asof("2014-01-02")
'2014-01-02'
If all of the labels in the index are later than the passed label,
NaN is returned.
- >>> idx.asof('1999-01-02')
+ >>> idx.asof("1999-01-02")
nan
If the index is not sorted, an error is raised.
- >>> idx_not_sorted = pd.Index(['2013-12-31', '2015-01-02',
- ... '2014-01-03'])
- >>> idx_not_sorted.asof('2013-12-31')
+ >>> idx_not_sorted = pd.Index(["2013-12-31", "2015-01-02", "2014-01-03"])
+ >>> idx_not_sorted.asof("2013-12-31")
Traceback (most recent call last):
ValueError: index must be monotonic increasing or decreasing
"""
self._searchsorted_monotonic(label) # validate sortedness
try:
loc = self.get_loc(label)
- except (KeyError, TypeError):
+ except (KeyError, TypeError) as err:
# KeyError -> No exact match, try for padded
# TypeError -> passed e.g. non-hashable, fall through to get
# the tested exception message
indexer = self.get_indexer([label], method="pad")
if indexer.ndim > 1 or indexer.size > 1:
- raise TypeError("asof requires scalar valued input")
+ raise TypeError("asof requires scalar valued input") from err
loc = indexer.item()
if loc == -1:
return self._na_value
@@ -5790,9 +5400,10 @@ def asof_locs(
Examples
--------
- >>> idx = pd.date_range('2023-06-01', periods=3, freq='D')
- >>> where = pd.DatetimeIndex(['2023-05-30 00:12:00', '2023-06-01 00:00:00',
- ... '2023-06-02 23:59:59'])
+ >>> idx = pd.date_range("2023-06-01", periods=3, freq="D")
+ >>> where = pd.DatetimeIndex(
+ ... ["2023-05-30 00:12:00", "2023-06-01 00:00:00", "2023-06-02 23:59:59"]
+ ... )
>>> mask = np.ones(3, dtype=bool)
>>> idx.asof_locs(where, mask)
array([-1, 0, 1])
@@ -5807,7 +5418,8 @@ def asof_locs(
# types "Union[ExtensionArray, ndarray[Any, Any]]", "str"
# TODO: will be fixed when ExtensionArray.searchsorted() is fixed
locs = self._values[mask].searchsorted(
- where._values, side="right" # type: ignore[call-overload]
+ where._values,
+ side="right", # type: ignore[call-overload]
)
locs = np.where(locs > 0, locs - 1, 0)
@@ -5826,8 +5438,7 @@ def sort_values(
ascending: bool = ...,
na_position: NaPosition = ...,
key: Callable | None = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def sort_values(
@@ -5837,8 +5448,7 @@ def sort_values(
ascending: bool = ...,
na_position: NaPosition = ...,
key: Callable | None = ...,
- ) -> tuple[Self, np.ndarray]:
- ...
+ ) -> tuple[Self, np.ndarray]: ...
@overload
def sort_values(
@@ -5848,14 +5458,11 @@ def sort_values(
ascending: bool = ...,
na_position: NaPosition = ...,
key: Callable | None = ...,
- ) -> Self | tuple[Self, np.ndarray]:
- ...
+ ) -> Self | tuple[Self, np.ndarray]: ...
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self"], name="sort_values"
- )
def sort_values(
self,
+ *,
return_indexer: bool = False,
ascending: bool = True,
na_position: NaPosition = "last",
@@ -5913,17 +5520,14 @@ def sort_values(
(Index([1000, 100, 10, 1], dtype='int64'), array([3, 1, 0, 2]))
"""
if key is None and (
- self.is_monotonic_increasing or self.is_monotonic_decreasing
+ (ascending and self.is_monotonic_increasing)
+ or (not ascending and self.is_monotonic_decreasing)
):
- reverse = ascending != self.is_monotonic_increasing
- sorted_index = self[::-1] if reverse else self.copy()
if return_indexer:
indexer = np.arange(len(self), dtype=np.intp)
- if reverse:
- indexer = indexer[::-1]
- return sorted_index, indexer
+ return self.copy(), indexer
else:
- return sorted_index
+ return self.copy()
# GH 35584. Sort missing values according to na_position kwarg
# ignore na_position for MultiIndex
@@ -5951,7 +5555,7 @@ def sort(self, *args, **kwargs):
"""
raise TypeError("cannot sort an Index object in-place, use sort_values instead")
- def shift(self, periods: int = 1, freq=None):
+ def shift(self, periods: int = 1, freq=None) -> Self:
"""
Shift index by desired number of time frequency increments.
@@ -5986,7 +5590,7 @@ def shift(self, periods: int = 1, freq=None):
--------
Put the first 5 month starts of 2011 into an index.
- >>> month_starts = pd.date_range('1/1/2011', periods=5, freq='MS')
+ >>> month_starts = pd.date_range("1/1/2011", periods=5, freq="MS")
>>> month_starts
DatetimeIndex(['2011-01-01', '2011-02-01', '2011-03-01', '2011-04-01',
'2011-05-01'],
@@ -5994,7 +5598,7 @@ def shift(self, periods: int = 1, freq=None):
Shift the index by 10 days.
- >>> month_starts.shift(10, freq='D')
+ >>> month_starts.shift(10, freq="D")
DatetimeIndex(['2011-01-11', '2011-02-11', '2011-03-11', '2011-04-11',
'2011-05-11'],
dtype='datetime64[ns]', freq=None)
@@ -6036,7 +5640,7 @@ def argsort(self, *args, **kwargs) -> npt.NDArray[np.intp]:
Examples
--------
- >>> idx = pd.Index(['b', 'a', 'd', 'c'])
+ >>> idx = pd.Index(["b", "a", "d", "c"])
>>> idx
Index(['b', 'a', 'd', 'c'], dtype='object')
@@ -6069,9 +5673,7 @@ def _should_fallback_to_positional(self) -> bool:
"complex",
}
- _index_shared_docs[
- "get_indexer_non_unique"
- ] = """
+ _index_shared_docs["get_indexer_non_unique"] = """
Compute indexer and mask for new index given the current index.
The indexer should be then used as an input to ndarray.take to align the
@@ -6173,7 +5775,7 @@ def get_indexer_for(self, target) -> npt.NDArray[np.intp]:
Examples
--------
- >>> idx = pd.Index([np.nan, 'var1', np.nan])
+ >>> idx = pd.Index([np.nan, "var1", np.nan])
>>> idx.get_indexer_for([np.nan])
array([0, 2])
"""
@@ -6253,20 +5855,17 @@ def _raise_if_missing(self, key, indexer, axis_name: str_t) -> None:
@overload
def _get_indexer_non_comparable(
self, target: Index, method, unique: Literal[True] = ...
- ) -> npt.NDArray[np.intp]:
- ...
+ ) -> npt.NDArray[np.intp]: ...
@overload
def _get_indexer_non_comparable(
self, target: Index, method, unique: Literal[False]
- ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]:
- ...
+ ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
@overload
def _get_indexer_non_comparable(
self, target: Index, method, unique: bool = True
- ) -> npt.NDArray[np.intp] | tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]:
- ...
+ ) -> npt.NDArray[np.intp] | tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]: ...
@final
def _get_indexer_non_comparable(
@@ -6424,7 +6023,7 @@ def _is_comparable_dtype(self, dtype: DtypeObj) -> bool:
return True
@final
- def groupby(self, values) -> PrettyDict[Hashable, np.ndarray]:
+ def groupby(self, values) -> PrettyDict[Hashable, Index]:
"""
Group the index labels by a given array of values.
@@ -6472,16 +6071,16 @@ def map(self, mapper, na_action: Literal["ignore"] | None = None):
Examples
--------
>>> idx = pd.Index([1, 2, 3])
- >>> idx.map({1: 'a', 2: 'b', 3: 'c'})
+ >>> idx.map({1: "a", 2: "b", 3: "c"})
Index(['a', 'b', 'c'], dtype='object')
Using `map` with a function:
>>> idx = pd.Index([1, 2, 3])
- >>> idx.map('I am a {}'.format)
+ >>> idx.map("I am a {}".format)
Index(['I am a 1', 'I am a 2', 'I am a 3'], dtype='object')
- >>> idx = pd.Index(['a', 'b', 'c'])
+ >>> idx = pd.Index(["a", "b", "c"])
>>> idx.map(lambda x: x.upper())
Index(['A', 'B', 'C'], dtype='object')
"""
@@ -6576,7 +6175,7 @@ def isin(self, values, level=None) -> npt.NDArray[np.bool_]:
Examples
--------
- >>> idx = pd.Index([1,2,3])
+ >>> idx = pd.Index([1, 2, 3])
>>> idx
Index([1, 2, 3], dtype='int64')
@@ -6585,24 +6184,24 @@ def isin(self, values, level=None) -> npt.NDArray[np.bool_]:
>>> idx.isin([1, 4])
array([ True, False, False])
- >>> midx = pd.MultiIndex.from_arrays([[1,2,3],
- ... ['red', 'blue', 'green']],
- ... names=('number', 'color'))
+ >>> midx = pd.MultiIndex.from_arrays(
+ ... [[1, 2, 3], ["red", "blue", "green"]], names=("number", "color")
+ ... )
>>> midx
MultiIndex([(1, 'red'),
(2, 'blue'),
(3, 'green')],
- names=['number', 'color'])
+ names=('number', 'color'))
Check whether the strings in the 'color' level of the MultiIndex
are in a list of colors.
- >>> midx.isin(['red', 'orange', 'yellow'], level='color')
+ >>> midx.isin(["red", "orange", "yellow"], level="color")
array([ True, False, False])
To check across the levels of a MultiIndex, pass a list of tuples:
- >>> midx.isin([(1, 'red'), (3, 'red')])
+ >>> midx.isin([(1, "red"), (3, "red")])
array([ True, False, False])
"""
if level is not None:
@@ -6650,12 +6249,12 @@ def slice_indexer(
--------
This is a method on all index types. For example you can do:
- >>> idx = pd.Index(list('abcd'))
- >>> idx.slice_indexer(start='b', end='c')
+ >>> idx = pd.Index(list("abcd"))
+ >>> idx.slice_indexer(start="b", end="c")
slice(1, 3, None)
- >>> idx = pd.MultiIndex.from_arrays([list('abcd'), list('efgh')])
- >>> idx.slice_indexer(start='b', end=('c', 'g'))
+ >>> idx = pd.MultiIndex.from_arrays([list("abcd"), list("efgh")])
+ >>> idx.slice_indexer(start="b", end=("c", "g"))
slice(1, 3, None)
"""
start_slice, end_slice = self.slice_locs(start, end, step=step)
@@ -6766,16 +6365,16 @@ def get_slice_bound(self, label, side: Literal["left", "right"]) -> int:
Examples
--------
>>> idx = pd.RangeIndex(5)
- >>> idx.get_slice_bound(3, 'left')
+ >>> idx.get_slice_bound(3, "left")
3
- >>> idx.get_slice_bound(3, 'right')
+ >>> idx.get_slice_bound(3, "right")
4
If ``label`` is non-unique in the index, an error will be raised.
- >>> idx_duplicate = pd.Index(['a', 'b', 'a', 'c', 'd'])
- >>> idx_duplicate.get_slice_bound('a', 'left')
+ >>> idx_duplicate = pd.Index(["a", "b", "a", "c", "d"])
+ >>> idx_duplicate.get_slice_bound("a", "left")
Traceback (most recent call last):
KeyError: Cannot get left slice bound for non-unique label: 'a'
"""
@@ -6800,7 +6399,7 @@ def get_slice_bound(self, label, side: Literal["left", "right"]) -> int:
return self._searchsorted_monotonic(label, side)
except ValueError:
# raise the original KeyError
- raise err
+ raise err from None
if isinstance(slc, np.ndarray):
# get_loc may return a boolean array, which
@@ -6810,7 +6409,7 @@ def get_slice_bound(self, label, side: Literal["left", "right"]) -> int:
if isinstance(slc, np.ndarray):
raise KeyError(
f"Cannot get {side} slice bound for non-unique "
- f"label: {repr(original_label)}"
+ f"label: {original_label!r}"
)
if isinstance(slc, slice):
@@ -6851,8 +6450,8 @@ def slice_locs(self, start=None, end=None, step=None) -> tuple[int, int]:
Examples
--------
- >>> idx = pd.Index(list('abcd'))
- >>> idx.slice_locs(start='b', end='c')
+ >>> idx = pd.Index(list("abcd"))
+ >>> idx.slice_locs(start="b", end="c")
(1, 3)
"""
inc = step is None or step >= 0
@@ -6933,11 +6532,11 @@ def delete(self, loc) -> Self:
Examples
--------
- >>> idx = pd.Index(['a', 'b', 'c'])
+ >>> idx = pd.Index(["a", "b", "c"])
>>> idx.delete(1)
Index(['a', 'c'], dtype='object')
- >>> idx = pd.Index(['a', 'b', 'c'])
+ >>> idx = pd.Index(["a", "b", "c"])
>>> idx.delete([0, 2])
Index(['b'], dtype='object')
"""
@@ -6969,8 +6568,8 @@ def insert(self, loc: int, item) -> Index:
Examples
--------
- >>> idx = pd.Index(['a', 'b', 'c'])
- >>> idx.insert(1, 'x')
+ >>> idx = pd.Index(["a", "b", "c"])
+ >>> idx.insert(1, "x")
Index(['a', 'x', 'b', 'c'], dtype='object')
"""
item = lib.item_from_zerodim(item)
@@ -7008,23 +6607,8 @@ def insert(self, loc: int, item) -> Index:
loc = loc if loc >= 0 else loc - 1
new_values[loc] = item
- out = Index._with_infer(new_values, name=self.name)
- if (
- using_pyarrow_string_dtype()
- and is_string_dtype(out.dtype)
- and new_values.dtype == object
- ):
- out = out.astype(new_values.dtype)
- if self.dtype == object and out.dtype != object:
- # GH#51363
- warnings.warn(
- "The behavior of Index.insert with object-dtype is deprecated, "
- "in a future version this will return an object-dtype Index "
- "instead of inferring a non-object dtype. To retain the old "
- "behavior, do `idx.insert(loc, item).infer_objects(copy=False)`",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
+ # GH#51363 stopped doing dtype inference here
+ out = Index(new_values, dtype=new_values.dtype, name=self.name)
return out
def drop(
@@ -7053,8 +6637,8 @@ def drop(
Examples
--------
- >>> idx = pd.Index(['a', 'b', 'c'])
- >>> idx.drop(['a'])
+ >>> idx = pd.Index(["a", "b", "c"])
+ >>> idx.drop(["a"])
Index(['b', 'c'], dtype='object')
"""
if not isinstance(labels, Index):
@@ -7131,7 +6715,6 @@ def diff(self, periods: int = 1) -> Index:
"""
return Index(self.to_series().diff(periods))
- @final
def round(self, decimals: int = 0) -> Self:
"""
Round each value in the Index to the given number of decimals.
@@ -7371,16 +6954,10 @@ def argmin(self, axis=None, skipna: bool = True, *args, **kwargs) -> int:
if not self._is_multi and self.hasnans:
# Take advantage of cache
- mask = self._isnan
- if not skipna or mask.all():
- warnings.warn(
- f"The behavior of {type(self).__name__}.argmax/argmin "
- "with skipna=False and NAs, or with all-NAs is deprecated. "
- "In a future version this will raise ValueError.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return -1
+ if self._isnan.all():
+ raise ValueError("Encountered all NA values")
+ elif not skipna:
+ raise ValueError("Encountered an NA value with skipna=False")
return super().argmin(skipna=skipna)
@Appender(IndexOpsMixin.argmax.__doc__)
@@ -7390,16 +6967,10 @@ def argmax(self, axis=None, skipna: bool = True, *args, **kwargs) -> int:
if not self._is_multi and self.hasnans:
# Take advantage of cache
- mask = self._isnan
- if not skipna or mask.all():
- warnings.warn(
- f"The behavior of {type(self).__name__}.argmax/argmin "
- "with skipna=False and NAs, or with all-NAs is deprecated. "
- "In a future version this will raise ValueError.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- return -1
+ if self._isnan.all():
+ raise ValueError("Encountered all NA values")
+ elif not skipna:
+ raise ValueError("Encountered an NA value with skipna=False")
return super().argmax(skipna=skipna)
def min(self, axis=None, skipna: bool = True, *args, **kwargs):
@@ -7432,13 +7003,13 @@ def min(self, axis=None, skipna: bool = True, *args, **kwargs):
>>> idx.min()
1
- >>> idx = pd.Index(['c', 'b', 'a'])
+ >>> idx = pd.Index(["c", "b", "a"])
>>> idx.min()
'a'
For a MultiIndex, the minimum is determined lexicographically.
- >>> idx = pd.MultiIndex.from_product([('a', 'b'), (2, 1)])
+ >>> idx = pd.MultiIndex.from_product([("a", "b"), (2, 1)])
>>> idx.min()
('a', 1)
"""
@@ -7495,13 +7066,13 @@ def max(self, axis=None, skipna: bool = True, *args, **kwargs):
>>> idx.max()
3
- >>> idx = pd.Index(['c', 'b', 'a'])
+ >>> idx = pd.Index(["c", "b", "a"])
>>> idx.max()
'c'
For a MultiIndex, the maximum is determined lexicographically.
- >>> idx = pd.MultiIndex.from_product([('a', 'b'), (2, 1)])
+ >>> idx = pd.MultiIndex.from_product([("a", "b"), (2, 1)])
>>> idx.max()
('b', 2)
"""
@@ -7549,6 +7120,36 @@ def shape(self) -> Shape:
return (len(self),)
+def maybe_sequence_to_range(sequence) -> Any | range:
+ """
+ Convert a 1D, non-pandas sequence to a range if possible.
+
+ Returns the input if not possible.
+
+ Parameters
+ ----------
+ sequence : 1D sequence
+ names : sequence of str
+
+ Returns
+ -------
+ Any : input or range
+ """
+ if isinstance(sequence, (ABCSeries, Index, range, ExtensionArray)):
+ return sequence
+ elif len(sequence) == 1 or lib.infer_dtype(sequence, skipna=False) != "integer":
+ return sequence
+ elif len(sequence) == 0:
+ return range(0)
+ diff = sequence[1] - sequence[0]
+ if diff == 0:
+ return sequence
+ elif len(sequence) == 2 or lib.is_sequence_range(np.asarray(sequence), diff):
+ return range(sequence[0], sequence[-1] + diff, diff)
+ else:
+ return sequence
+
+
def ensure_index_from_sequences(sequences, names=None) -> Index:
"""
Construct an index from sequences of data.
@@ -7567,13 +7168,13 @@ def ensure_index_from_sequences(sequences, names=None) -> Index:
Examples
--------
- >>> ensure_index_from_sequences([[1, 2, 3]], names=["name"])
- Index([1, 2, 3], dtype='int64', name='name')
+ >>> ensure_index_from_sequences([[1, 2, 4]], names=["name"])
+ Index([1, 2, 4], dtype='int64', name='name')
>>> ensure_index_from_sequences([["a", "a"], ["a", "b"]], names=["L1", "L2"])
MultiIndex([('a', 'a'),
('a', 'b')],
- names=['L1', 'L2'])
+ names=('L1', 'L2'))
See Also
--------
@@ -7584,8 +7185,9 @@ def ensure_index_from_sequences(sequences, names=None) -> Index:
if len(sequences) == 1:
if names is not None:
names = names[0]
- return Index(sequences[0], name=names)
+ return Index(maybe_sequence_to_range(sequences[0]), name=names)
else:
+ # TODO: Apply maybe_sequence_to_range to sequences?
return MultiIndex.from_arrays(sequences, names=names)
@@ -7609,13 +7211,13 @@ def ensure_index(index_like: Axes, copy: bool = False) -> Index:
Examples
--------
- >>> ensure_index(['a', 'b'])
+ >>> ensure_index(["a", "b"])
Index(['a', 'b'], dtype='object')
- >>> ensure_index([('a', 'a'), ('b', 'c')])
+ >>> ensure_index([("a", "a"), ("b", "c")])
Index([('a', 'a'), ('b', 'c')], dtype='object')
- >>> ensure_index([['a', 'a'], ['b', 'c']])
+ >>> ensure_index([["a", "a"], ["b", "c"]])
MultiIndex([('a', 'b'),
('a', 'c')],
)
diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py
index b307be004ad6e..3b04d95cb7cbd 100644
--- a/pandas/core/indexes/category.py
+++ b/pandas/core/indexes/category.py
@@ -276,22 +276,34 @@ def equals(self, other: object) -> bool:
"""
Determine if two CategoricalIndex objects contain the same elements.
+ The order and orderedness of elements matters. The categories matter,
+ but the order of the categories matters only when ``ordered=True``.
+
+ Parameters
+ ----------
+ other : object
+ The CategoricalIndex object to compare with.
+
Returns
-------
bool
``True`` if two :class:`pandas.CategoricalIndex` objects have equal
elements, ``False`` otherwise.
+ See Also
+ --------
+ Categorical.equals : Returns True if categorical arrays are equal.
+
Examples
--------
- >>> ci = pd.CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'])
- >>> ci2 = pd.CategoricalIndex(pd.Categorical(['a', 'b', 'c', 'a', 'b', 'c']))
+ >>> ci = pd.CategoricalIndex(["a", "b", "c", "a", "b", "c"])
+ >>> ci2 = pd.CategoricalIndex(pd.Categorical(["a", "b", "c", "a", "b", "c"]))
>>> ci.equals(ci2)
True
The order of elements matters.
- >>> ci3 = pd.CategoricalIndex(['c', 'b', 'a', 'a', 'b', 'c'])
+ >>> ci3 = pd.CategoricalIndex(["c", "b", "a", "a", "b", "c"])
>>> ci.equals(ci3)
False
@@ -304,16 +316,17 @@ def equals(self, other: object) -> bool:
The categories matter, but the order of the categories matters only when
``ordered=True``.
- >>> ci5 = ci.set_categories(['a', 'b', 'c', 'd'])
+ >>> ci5 = ci.set_categories(["a", "b", "c", "d"])
>>> ci.equals(ci5)
False
- >>> ci6 = ci.set_categories(['b', 'c', 'a'])
+ >>> ci6 = ci.set_categories(["b", "c", "a"])
>>> ci.equals(ci6)
True
- >>> ci_ordered = pd.CategoricalIndex(['a', 'b', 'c', 'a', 'b', 'c'],
- ... ordered=True)
- >>> ci2_ordered = ci_ordered.set_categories(['b', 'c', 'a'])
+ >>> ci_ordered = pd.CategoricalIndex(
+ ... ["a", "b", "c", "a", "b", "c"], ordered=True
+ ... )
+ >>> ci2_ordered = ci_ordered.set_categories(["b", "c", "a"])
>>> ci_ordered.equals(ci2_ordered)
False
"""
@@ -445,6 +458,9 @@ def map(self, mapper, na_action: Literal["ignore"] | None = None):
----------
mapper : function, dict, or Series
Mapping correspondence.
+ na_action : {None, 'ignore'}, default 'ignore'
+ If 'ignore', propagate NaN values, without passing them to
+ the mapping correspondence.
Returns
-------
@@ -462,37 +478,37 @@ def map(self, mapper, na_action: Literal["ignore"] | None = None):
Examples
--------
- >>> idx = pd.CategoricalIndex(['a', 'b', 'c'])
+ >>> idx = pd.CategoricalIndex(["a", "b", "c"])
>>> idx
CategoricalIndex(['a', 'b', 'c'], categories=['a', 'b', 'c'],
ordered=False, dtype='category')
>>> idx.map(lambda x: x.upper())
CategoricalIndex(['A', 'B', 'C'], categories=['A', 'B', 'C'],
ordered=False, dtype='category')
- >>> idx.map({'a': 'first', 'b': 'second', 'c': 'third'})
+ >>> idx.map({"a": "first", "b": "second", "c": "third"})
CategoricalIndex(['first', 'second', 'third'], categories=['first',
'second', 'third'], ordered=False, dtype='category')
If the mapping is one-to-one the ordering of the categories is
preserved:
- >>> idx = pd.CategoricalIndex(['a', 'b', 'c'], ordered=True)
+ >>> idx = pd.CategoricalIndex(["a", "b", "c"], ordered=True)
>>> idx
CategoricalIndex(['a', 'b', 'c'], categories=['a', 'b', 'c'],
ordered=True, dtype='category')
- >>> idx.map({'a': 3, 'b': 2, 'c': 1})
+ >>> idx.map({"a": 3, "b": 2, "c": 1})
CategoricalIndex([3, 2, 1], categories=[3, 2, 1], ordered=True,
dtype='category')
If the mapping is not one-to-one an :class:`~pandas.Index` is returned:
- >>> idx.map({'a': 'first', 'b': 'second', 'c': 'first'})
+ >>> idx.map({"a": "first", "b": "second", "c": "first"})
Index(['first', 'second', 'first'], dtype='object')
If a `dict` is used, all unmapped categories are mapped to `NaN` and
the result is an :class:`~pandas.Index`:
- >>> idx.map({'a': 'first', 'b': 'second'})
+ >>> idx.map({"a": "first", "b": "second"})
Index(['first', 'second', nan], dtype='object')
"""
mapped = self._values.map(mapper, na_action=na_action)
diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py
index cad8737a987d4..7e8d808769bc1 100644
--- a/pandas/core/indexes/datetimelike.py
+++ b/pandas/core/indexes/datetimelike.py
@@ -1,6 +1,7 @@
"""
Base and utility classes for tseries type pandas objects.
"""
+
from __future__ import annotations
from abc import (
@@ -10,16 +11,12 @@
from typing import (
TYPE_CHECKING,
Any,
- Callable,
cast,
final,
)
-import warnings
import numpy as np
-from pandas._config import using_copy_on_write
-
from pandas._libs import (
NaT,
Timedelta,
@@ -32,7 +29,6 @@
parsing,
to_offset,
)
-from pandas._libs.tslibs.dtypes import freq_to_period_freqstr
from pandas.compat.numpy import function as nv
from pandas.errors import (
InvalidIndexError,
@@ -43,14 +39,16 @@
cache_readonly,
doc,
)
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.common import (
is_integer,
is_list_like,
)
from pandas.core.dtypes.concat import concat_compat
-from pandas.core.dtypes.dtypes import CategoricalDtype
+from pandas.core.dtypes.dtypes import (
+ CategoricalDtype,
+ PeriodDtype,
+)
from pandas.core.arrays import (
DatetimeArray,
@@ -75,6 +73,7 @@
from pandas._typing import (
Axis,
+ JoinHow,
Self,
npt,
)
@@ -98,6 +97,32 @@ def mean(self, *, skipna: bool = True, axis: int | None = 0):
@property
def freq(self) -> BaseOffset | None:
+ """
+ Return the frequency object if it is set, otherwise None.
+
+ To learn more about the frequency strings, please see
+ :ref:`this link`.
+
+ See Also
+ --------
+ DatetimeIndex.freq : Return the frequency object if it is set, otherwise None.
+ PeriodIndex.freq : Return the frequency object if it is set, otherwise None.
+
+ Examples
+ --------
+ >>> datetimeindex = pd.date_range(
+ ... "2022-02-22 02:22:22", periods=10, tz="America/Chicago", freq="h"
+ ... )
+ >>> datetimeindex
+ DatetimeIndex(['2022-02-22 02:22:22-06:00', '2022-02-22 03:22:22-06:00',
+ '2022-02-22 04:22:22-06:00', '2022-02-22 05:22:22-06:00',
+ '2022-02-22 06:22:22-06:00', '2022-02-22 07:22:22-06:00',
+ '2022-02-22 08:22:22-06:00', '2022-02-22 09:22:22-06:00',
+ '2022-02-22 10:22:22-06:00', '2022-02-22 11:22:22-06:00'],
+ dtype='datetime64[ns, America/Chicago]', freq='h')
+ >>> datetimeindex.freq
+
+ """
return self._data.freq
@freq.setter
@@ -117,15 +142,14 @@ def freqstr(self) -> str:
if self._data.freqstr is not None and isinstance(
self._data, (PeriodArray, PeriodIndex)
):
- freq = freq_to_period_freqstr(self._data.freq.n, self._data.freq.name)
+ freq = PeriodDtype(self._data.freq)._freqstr
return freq
else:
return self._data.freqstr # type: ignore[return-value]
@cache_readonly
@abstractmethod
- def _resolution_obj(self) -> Resolution:
- ...
+ def _resolution_obj(self) -> Resolution: ...
@cache_readonly
@doc(DatetimeLikeArrayMixin.resolution)
@@ -191,39 +215,6 @@ def _convert_tolerance(self, tolerance, target):
# Rendering Methods
_default_na_rep = "NaT"
- def format(
- self,
- name: bool = False,
- formatter: Callable | None = None,
- na_rep: str = "NaT",
- date_format: str | None = None,
- ) -> list[str]:
- """
- Render a string representation of the Index.
- """
- warnings.warn(
- # GH#55413
- f"{type(self).__name__}.format is deprecated and will be removed "
- "in a future version. Convert using index.astype(str) or "
- "index.map(formatter) instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- header = []
- if name:
- header.append(
- ibase.pprint_thing(self.name, escape_chars=("\t", "\r", "\n"))
- if self.name is not None
- else ""
- )
-
- if formatter is not None:
- return header + list(self.map(formatter))
-
- return self._format_with_header(
- header=header, na_rep=na_rep, date_format=date_format
- )
-
def _format_with_header(
self, *, header: list[str], na_rep: str, date_format: str | None = None
) -> list[str]:
@@ -272,7 +263,7 @@ def _can_partial_date_slice(self, reso: Resolution) -> bool:
def _parsed_string_to_bounds(self, reso: Resolution, parsed):
raise NotImplementedError
- def _parse_with_reso(self, label: str):
+ def _parse_with_reso(self, label: str) -> tuple[datetime, Resolution]:
# overridden by TimedeltaIndex
try:
if self.freq is None or hasattr(self.freq, "rule_code"):
@@ -294,7 +285,7 @@ def _parse_with_reso(self, label: str):
reso = Resolution.from_attrname(reso_str)
return parsed, reso
- def _get_string_slice(self, key: str):
+ def _get_string_slice(self, key: str) -> slice | npt.NDArray[np.intp]:
# overridden by TimedeltaIndex
parsed, reso = self._parse_with_reso(key)
try:
@@ -462,20 +453,20 @@ def as_unit(self, unit: str) -> Self:
--------
For :class:`pandas.DatetimeIndex`:
- >>> idx = pd.DatetimeIndex(['2020-01-02 01:02:03.004005006'])
+ >>> idx = pd.DatetimeIndex(["2020-01-02 01:02:03.004005006"])
>>> idx
DatetimeIndex(['2020-01-02 01:02:03.004005006'],
dtype='datetime64[ns]', freq=None)
- >>> idx.as_unit('s')
+ >>> idx.as_unit("s")
DatetimeIndex(['2020-01-02 01:02:03'], dtype='datetime64[s]', freq=None)
For :class:`pandas.TimedeltaIndex`:
- >>> tdelta_idx = pd.to_timedelta(['1 day 3 min 2 us 42 ns'])
+ >>> tdelta_idx = pd.to_timedelta(["1 day 3 min 2 us 42 ns"])
>>> tdelta_idx
TimedeltaIndex(['1 days 00:03:00.000002042'],
dtype='timedelta64[ns]', freq=None)
- >>> tdelta_idx.as_unit('s')
+ >>> tdelta_idx.as_unit("s")
TimedeltaIndex(['1 days 00:03:00'], dtype='timedelta64[s]', freq=None)
"""
arr = self._data.as_unit(unit)
@@ -489,9 +480,8 @@ def _with_freq(self, freq):
def values(self) -> np.ndarray:
# NB: For Datetime64TZ this is lossy
data = self._data._ndarray
- if using_copy_on_write():
- data = data.view()
- data.flags.writeable = False
+ data = data.view()
+ data.flags.writeable = False
return data
@doc(DatetimeIndexOpsMixin.shift)
@@ -734,13 +724,20 @@ def _get_join_freq(self, other):
freq = self.freq
return freq
- def _wrap_joined_index(
- self, joined, other, lidx: npt.NDArray[np.intp], ridx: npt.NDArray[np.intp]
- ):
+ def _wrap_join_result(
+ self,
+ joined,
+ other,
+ lidx: npt.NDArray[np.intp] | None,
+ ridx: npt.NDArray[np.intp] | None,
+ how: JoinHow,
+ ) -> tuple[Self, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
assert other.dtype == self.dtype, (other.dtype, self.dtype)
- result = super()._wrap_joined_index(joined, other, lidx, ridx)
- result._data._freq = self._get_join_freq(other)
- return result
+ join_index, lidx, ridx = super()._wrap_join_result(
+ joined, other, lidx, ridx, how
+ )
+ join_index._data._freq = self._get_join_freq(other)
+ return join_index, lidx, ridx
def _get_engine_target(self) -> np.ndarray:
# engine methods and libjoin methods need dt64/td64 values cast to i8
diff --git a/pandas/core/indexes/datetimes.py b/pandas/core/indexes/datetimes.py
index c978abd8c2427..cefdc14145d1f 100644
--- a/pandas/core/indexes/datetimes.py
+++ b/pandas/core/indexes/datetimes.py
@@ -28,7 +28,6 @@
cache_readonly,
doc,
)
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.common import is_scalar
from pandas.core.dtypes.dtypes import DatetimeTZDtype
@@ -150,17 +149,6 @@ class DatetimeIndex(DatetimeTimedeltaMixin):
inferred frequency upon creation.
tz : pytz.timezone or dateutil.tz.tzfile or datetime.tzinfo or str
Set the Timezone of the data.
- normalize : bool, default False
- Normalize start/end dates to midnight before generating date range.
-
- .. deprecated:: 2.1.0
-
- closed : {'left', 'right'}, optional
- Set whether to include `start` and `end` that are on the
- boundary. The default includes boundary points on either end.
-
- .. deprecated:: 2.1.0
-
ambiguous : 'infer', bool-ndarray, 'NaT', default 'raise'
When clocks moved backward due to DST, ambiguous times may arise.
For example in Central European Time (UTC+01), when going from 03:00
@@ -246,8 +234,8 @@ class DatetimeIndex(DatetimeTimedeltaMixin):
Notes
-----
- To learn more about the frequency strings, please see `this link
- `__.
+ To learn more about the frequency strings, please see
+ :ref:`this link`.
Examples
--------
@@ -322,8 +310,6 @@ def __new__(
data=None,
freq: Frequency | lib.NoDefault = lib.no_default,
tz=lib.no_default,
- normalize: bool | lib.NoDefault = lib.no_default,
- closed=lib.no_default,
ambiguous: TimeAmbiguous = "raise",
dayfirst: bool = False,
yearfirst: bool = False,
@@ -331,23 +317,6 @@ def __new__(
copy: bool = False,
name: Hashable | None = None,
) -> Self:
- if closed is not lib.no_default:
- # GH#52628
- warnings.warn(
- f"The 'closed' keyword in {cls.__name__} construction is "
- "deprecated and will be removed in a future version.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- if normalize is not lib.no_default:
- # GH#52628
- warnings.warn(
- f"The 'normalize' keyword in {cls.__name__} construction is "
- "deprecated and will be removed in a future version.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
if is_scalar(data):
cls._raise_scalar_data_error(data)
@@ -488,12 +457,13 @@ def snap(self, freq: Frequency = "S") -> DatetimeIndex:
Examples
--------
- >>> idx = pd.DatetimeIndex(['2023-01-01', '2023-01-02',
- ... '2023-02-01', '2023-02-02'])
+ >>> idx = pd.DatetimeIndex(
+ ... ["2023-01-01", "2023-01-02", "2023-02-01", "2023-02-02"]
+ ... )
>>> idx
DatetimeIndex(['2023-01-01', '2023-01-02', '2023-02-01', '2023-02-02'],
dtype='datetime64[ns]', freq=None)
- >>> idx.snap('MS')
+ >>> idx.snap("MS")
DatetimeIndex(['2023-01-01', '2023-01-01', '2023-02-01', '2023-02-01'],
dtype='datetime64[ns]', freq=None)
"""
@@ -518,7 +488,9 @@ def snap(self, freq: Frequency = "S") -> DatetimeIndex:
# --------------------------------------------------------------------
# Indexing Methods
- def _parsed_string_to_bounds(self, reso: Resolution, parsed: dt.datetime):
+ def _parsed_string_to_bounds(
+ self, reso: Resolution, parsed: dt.datetime
+ ) -> tuple[Timestamp, Timestamp]:
"""
Calculate datetime bounds for parsed time string and its resolution.
@@ -555,7 +527,7 @@ def _parsed_string_to_bounds(self, reso: Resolution, parsed: dt.datetime):
# which localizes parsed.
return start, end
- def _parse_with_reso(self, label: str):
+ def _parse_with_reso(self, label: str) -> tuple[Timestamp, Resolution]:
parsed, reso = super()._parse_with_reso(label)
parsed = Timestamp(parsed)
@@ -735,8 +707,9 @@ def indexer_at_time(self, time, asof: bool = False) -> npt.NDArray[np.intp]:
Examples
--------
- >>> idx = pd.DatetimeIndex(["1/1/2020 10:00", "2/1/2020 11:00",
- ... "3/1/2020 10:00"])
+ >>> idx = pd.DatetimeIndex(
+ ... ["1/1/2020 10:00", "2/1/2020 11:00", "3/1/2020 10:00"]
+ ... )
>>> idx.indexer_at_time("10:00")
array([0, 2])
"""
@@ -770,7 +743,9 @@ def indexer_between_time(
appropriate format ("%H:%M", "%H%M", "%I:%M%p", "%I%M%p",
"%H:%M:%S", "%H%M%S", "%I:%M:%S%p","%I%M%S%p").
include_start : bool, default True
+ Include boundaries; whether to set start bound as closed or open.
include_end : bool, default True
+ Include boundaries; whether to set end bound as closed or open.
Returns
-------
@@ -835,13 +810,15 @@ def date_range(
Return a fixed frequency DatetimeIndex.
Returns the range of equally spaced time points (where the difference between any
- two adjacent points is specified by the given frequency) such that they all
- satisfy `start <[=] x <[=] end`, where the first one and the last one are, resp.,
- the first and last time points in that range that fall on the boundary of ``freq``
- (if given as a frequency string) or that are valid for ``freq`` (if given as a
- :class:`pandas.tseries.offsets.DateOffset`). (If exactly one of ``start``,
- ``end``, or ``freq`` is *not* specified, this missing parameter can be computed
- given ``periods``, the number of timesteps in the range. See the note below.)
+ two adjacent points is specified by the given frequency) such that they fall in the
+ range `[start, end]` , where the first one and the last one are, resp., the first
+ and last time points in that range that fall on the boundary of ``freq`` (if given
+ as a frequency string) or that are valid for ``freq`` (if given as a
+ :class:`pandas.tseries.offsets.DateOffset`). If ``freq`` is positive, the points
+ satisfy `start <[=] x <[=] end`, and if ``freq`` is negative, the points satisfy
+ `end <[=] x <[=] start`. (If exactly one of ``start``, ``end``, or ``freq`` is *not*
+ specified, this missing parameter can be computed given ``periods``, the number of
+ timesteps in the range. See the note below.)
Parameters
----------
@@ -892,8 +869,8 @@ def date_range(
``DatetimeIndex`` will have ``periods`` linearly spaced elements between
``start`` and ``end`` (closed on both sides).
- To learn more about the frequency strings, please see `this link
- `__.
+ To learn more about the frequency strings, please see
+ :ref:`this link`.
Examples
--------
@@ -904,7 +881,7 @@ def date_range(
Specify `start` and `end`, with the default daily frequency.
- >>> pd.date_range(start='1/1/2018', end='1/08/2018')
+ >>> pd.date_range(start="1/1/2018", end="1/08/2018")
DatetimeIndex(['2018-01-01', '2018-01-02', '2018-01-03', '2018-01-04',
'2018-01-05', '2018-01-06', '2018-01-07', '2018-01-08'],
dtype='datetime64[ns]', freq='D')
@@ -923,14 +900,14 @@ def date_range(
Specify `start` and `periods`, the number of periods (days).
- >>> pd.date_range(start='1/1/2018', periods=8)
+ >>> pd.date_range(start="1/1/2018", periods=8)
DatetimeIndex(['2018-01-01', '2018-01-02', '2018-01-03', '2018-01-04',
'2018-01-05', '2018-01-06', '2018-01-07', '2018-01-08'],
dtype='datetime64[ns]', freq='D')
Specify `end` and `periods`, the number of periods (days).
- >>> pd.date_range(end='1/1/2018', periods=8)
+ >>> pd.date_range(end="1/1/2018", periods=8)
DatetimeIndex(['2017-12-25', '2017-12-26', '2017-12-27', '2017-12-28',
'2017-12-29', '2017-12-30', '2017-12-31', '2018-01-01'],
dtype='datetime64[ns]', freq='D')
@@ -938,7 +915,7 @@ def date_range(
Specify `start`, `end`, and `periods`; the frequency is generated
automatically (linearly spaced).
- >>> pd.date_range(start='2018-04-24', end='2018-04-27', periods=3)
+ >>> pd.date_range(start="2018-04-24", end="2018-04-27", periods=3)
DatetimeIndex(['2018-04-24 00:00:00', '2018-04-25 12:00:00',
'2018-04-27 00:00:00'],
dtype='datetime64[ns]', freq=None)
@@ -947,28 +924,28 @@ def date_range(
Changed the `freq` (frequency) to ``'ME'`` (month end frequency).
- >>> pd.date_range(start='1/1/2018', periods=5, freq='ME')
+ >>> pd.date_range(start="1/1/2018", periods=5, freq="ME")
DatetimeIndex(['2018-01-31', '2018-02-28', '2018-03-31', '2018-04-30',
'2018-05-31'],
dtype='datetime64[ns]', freq='ME')
Multiples are allowed
- >>> pd.date_range(start='1/1/2018', periods=5, freq='3ME')
+ >>> pd.date_range(start="1/1/2018", periods=5, freq="3ME")
DatetimeIndex(['2018-01-31', '2018-04-30', '2018-07-31', '2018-10-31',
'2019-01-31'],
dtype='datetime64[ns]', freq='3ME')
`freq` can also be specified as an Offset object.
- >>> pd.date_range(start='1/1/2018', periods=5, freq=pd.offsets.MonthEnd(3))
+ >>> pd.date_range(start="1/1/2018", periods=5, freq=pd.offsets.MonthEnd(3))
DatetimeIndex(['2018-01-31', '2018-04-30', '2018-07-31', '2018-10-31',
'2019-01-31'],
dtype='datetime64[ns]', freq='3ME')
Specify `tz` to set the timezone.
- >>> pd.date_range(start='1/1/2018', periods=5, tz='Asia/Tokyo')
+ >>> pd.date_range(start="1/1/2018", periods=5, tz="Asia/Tokyo")
DatetimeIndex(['2018-01-01 00:00:00+09:00', '2018-01-02 00:00:00+09:00',
'2018-01-03 00:00:00+09:00', '2018-01-04 00:00:00+09:00',
'2018-01-05 00:00:00+09:00'],
@@ -977,20 +954,20 @@ def date_range(
`inclusive` controls whether to include `start` and `end` that are on the
boundary. The default, "both", includes boundary points on either end.
- >>> pd.date_range(start='2017-01-01', end='2017-01-04', inclusive="both")
+ >>> pd.date_range(start="2017-01-01", end="2017-01-04", inclusive="both")
DatetimeIndex(['2017-01-01', '2017-01-02', '2017-01-03', '2017-01-04'],
dtype='datetime64[ns]', freq='D')
Use ``inclusive='left'`` to exclude `end` if it falls on the boundary.
- >>> pd.date_range(start='2017-01-01', end='2017-01-04', inclusive='left')
+ >>> pd.date_range(start="2017-01-01", end="2017-01-04", inclusive="left")
DatetimeIndex(['2017-01-01', '2017-01-02', '2017-01-03'],
dtype='datetime64[ns]', freq='D')
Use ``inclusive='right'`` to exclude `start` if it falls on the boundary, and
similarly ``inclusive='neither'`` will exclude both `start` and `end`.
- >>> pd.date_range(start='2017-01-01', end='2017-01-04', inclusive='right')
+ >>> pd.date_range(start="2017-01-01", end="2017-01-04", inclusive="right")
DatetimeIndex(['2017-01-02', '2017-01-03', '2017-01-04'],
dtype='datetime64[ns]', freq='D')
@@ -1079,14 +1056,14 @@ def bdate_range(
for ``bdate_range``. Use ``date_range`` if specifying ``freq`` is not
desired.
- To learn more about the frequency strings, please see `this link
- `__.
+ To learn more about the frequency strings, please see
+ :ref:`this link`.
Examples
--------
Note how the two weekend days are skipped in the result.
- >>> pd.bdate_range(start='1/1/2018', end='1/08/2018')
+ >>> pd.bdate_range(start="1/1/2018", end="1/08/2018")
DatetimeIndex(['2018-01-01', '2018-01-02', '2018-01-03', '2018-01-04',
'2018-01-05', '2018-01-08'],
dtype='datetime64[ns]', freq='B')
diff --git a/pandas/core/indexes/extension.py b/pandas/core/indexes/extension.py
index 61949531f37df..fc806a3546571 100644
--- a/pandas/core/indexes/extension.py
+++ b/pandas/core/indexes/extension.py
@@ -1,8 +1,10 @@
"""
Shared methods for Index subclasses backed by ExtensionArray.
"""
+
from __future__ import annotations
+from inspect import signature
from typing import (
TYPE_CHECKING,
Callable,
@@ -104,6 +106,7 @@ def method(self, *args, **kwargs): # type: ignore[misc]
# error: "property" has no attribute "__name__"
method.__name__ = name # type: ignore[attr-defined]
method.__doc__ = attr.__doc__
+ method.__signature__ = signature(attr) # type: ignore[attr-defined]
return method
diff --git a/pandas/core/indexes/frozen.py b/pandas/core/indexes/frozen.py
deleted file mode 100644
index 9d528d34e3684..0000000000000
--- a/pandas/core/indexes/frozen.py
+++ /dev/null
@@ -1,120 +0,0 @@
-"""
-frozen (immutable) data structures to support MultiIndexing
-
-These are used for:
-
-- .names (FrozenList)
-
-"""
-from __future__ import annotations
-
-from typing import (
- TYPE_CHECKING,
- NoReturn,
-)
-
-from pandas.core.base import PandasObject
-
-from pandas.io.formats.printing import pprint_thing
-
-if TYPE_CHECKING:
- from pandas._typing import Self
-
-
-class FrozenList(PandasObject, list):
- """
- Container that doesn't allow setting item *but*
- because it's technically hashable, will be used
- for lookups, appropriately, etc.
- """
-
- # Side note: This has to be of type list. Otherwise,
- # it messes up PyTables type checks.
-
- def union(self, other) -> FrozenList:
- """
- Returns a FrozenList with other concatenated to the end of self.
-
- Parameters
- ----------
- other : array-like
- The array-like whose elements we are concatenating.
-
- Returns
- -------
- FrozenList
- The collection difference between self and other.
- """
- if isinstance(other, tuple):
- other = list(other)
- return type(self)(super().__add__(other))
-
- def difference(self, other) -> FrozenList:
- """
- Returns a FrozenList with elements from other removed from self.
-
- Parameters
- ----------
- other : array-like
- The array-like whose elements we are removing self.
-
- Returns
- -------
- FrozenList
- The collection difference between self and other.
- """
- other = set(other)
- temp = [x for x in self if x not in other]
- return type(self)(temp)
-
- # TODO: Consider deprecating these in favor of `union` (xref gh-15506)
- # error: Incompatible types in assignment (expression has type
- # "Callable[[FrozenList, Any], FrozenList]", base class "list" defined the
- # type as overloaded function)
- __add__ = __iadd__ = union # type: ignore[assignment]
-
- def __getitem__(self, n):
- if isinstance(n, slice):
- return type(self)(super().__getitem__(n))
- return super().__getitem__(n)
-
- def __radd__(self, other) -> Self:
- if isinstance(other, tuple):
- other = list(other)
- return type(self)(other + list(self))
-
- def __eq__(self, other: object) -> bool:
- if isinstance(other, (tuple, FrozenList)):
- other = list(other)
- return super().__eq__(other)
-
- __req__ = __eq__
-
- def __mul__(self, other) -> Self:
- return type(self)(super().__mul__(other))
-
- __imul__ = __mul__
-
- def __reduce__(self):
- return type(self), (list(self),)
-
- # error: Signature of "__hash__" incompatible with supertype "list"
- def __hash__(self) -> int: # type: ignore[override]
- return hash(tuple(self))
-
- def _disabled(self, *args, **kwargs) -> NoReturn:
- """
- This method will not function because object is immutable.
- """
- raise TypeError(f"'{type(self).__name__}' does not support mutable operations.")
-
- def __str__(self) -> str:
- return pprint_thing(self, quote_strings=True, escape_chars=("\t", "\r", "\n"))
-
- def __repr__(self) -> str:
- return f"{type(self).__name__}({str(self)})"
-
- __setitem__ = __setslice__ = _disabled # type: ignore[assignment]
- __delitem__ = __delslice__ = _disabled
- pop = append = extend = _disabled
- remove = sort = insert = _disabled # type: ignore[assignment]
diff --git a/pandas/core/indexes/interval.py b/pandas/core/indexes/interval.py
index 4fcdb87974511..36f181110eccd 100644
--- a/pandas/core/indexes/interval.py
+++ b/pandas/core/indexes/interval.py
@@ -1,4 +1,5 @@
-""" define the IntervalIndex """
+"""define the IntervalIndex"""
+
from __future__ import annotations
from operator import (
@@ -128,7 +129,7 @@ def _get_next_label(label):
elif is_float_dtype(dtype):
return np.nextafter(label, np.inf)
else:
- raise TypeError(f"cannot determine next label for type {repr(type(label))}")
+ raise TypeError(f"cannot determine next label for type {type(label)!r}")
def _get_prev_label(label):
@@ -145,7 +146,7 @@ def _get_prev_label(label):
elif is_float_dtype(dtype):
return np.nextafter(label, -np.inf)
else:
- raise TypeError(f"cannot determine next label for type {repr(type(label))}")
+ raise TypeError(f"cannot determine next label for type {type(label)!r}")
def _new_IntervalIndex(cls, d):
@@ -479,7 +480,7 @@ def is_overlapping(self) -> bool:
Intervals that share closed endpoints overlap:
- >>> index = pd.interval_range(0, 3, closed='both')
+ >>> index = pd.interval_range(0, 3, closed="both")
>>> index
IntervalIndex([[0, 1], [1, 2], [2, 3]],
dtype='interval[int64, both]')
@@ -488,7 +489,7 @@ def is_overlapping(self) -> bool:
Intervals that only have an open endpoint in common do not overlap:
- >>> index = pd.interval_range(0, 3, closed='left')
+ >>> index = pd.interval_range(0, 3, closed="left")
>>> index
IntervalIndex([[0, 1), [1, 2), [2, 3)],
dtype='interval[int64, left]')
@@ -555,9 +556,7 @@ def _maybe_convert_i8(self, key):
right = self._maybe_convert_i8(key.right)
constructor = Interval if scalar else IntervalIndex.from_arrays
# error: "object" not callable
- return constructor(
- left, right, closed=self.closed
- ) # type: ignore[operator]
+ return constructor(left, right, closed=self.closed) # type: ignore[operator]
if scalar:
# Timestamp/Timedelta
@@ -1006,8 +1005,8 @@ def interval_range(
``IntervalIndex`` will have ``periods`` linearly spaced elements between
``start`` and ``end``, inclusively.
- To learn more about datetime-like frequency strings, please see `this link
- `__.
+ To learn more about datetime-like frequency strings, please see
+ :ref:`this link`.
Examples
--------
@@ -1019,8 +1018,9 @@ def interval_range(
Additionally, datetime-like input is also supported.
- >>> pd.interval_range(start=pd.Timestamp('2017-01-01'),
- ... end=pd.Timestamp('2017-01-04'))
+ >>> pd.interval_range(
+ ... start=pd.Timestamp("2017-01-01"), end=pd.Timestamp("2017-01-04")
+ ... )
IntervalIndex([(2017-01-01 00:00:00, 2017-01-02 00:00:00],
(2017-01-02 00:00:00, 2017-01-03 00:00:00],
(2017-01-03 00:00:00, 2017-01-04 00:00:00]],
@@ -1037,8 +1037,7 @@ def interval_range(
Similarly, for datetime-like ``start`` and ``end``, the frequency must be
convertible to a DateOffset.
- >>> pd.interval_range(start=pd.Timestamp('2017-01-01'),
- ... periods=3, freq='MS')
+ >>> pd.interval_range(start=pd.Timestamp("2017-01-01"), periods=3, freq="MS")
IntervalIndex([(2017-01-01 00:00:00, 2017-02-01 00:00:00],
(2017-02-01 00:00:00, 2017-03-01 00:00:00],
(2017-03-01 00:00:00, 2017-04-01 00:00:00]],
@@ -1054,7 +1053,7 @@ def interval_range(
The ``closed`` parameter specifies which endpoints of the individual
intervals within the ``IntervalIndex`` are closed.
- >>> pd.interval_range(end=5, periods=4, closed='both')
+ >>> pd.interval_range(end=5, periods=4, closed="both")
IntervalIndex([[1, 2], [2, 3], [3, 4], [4, 5]],
dtype='interval[int64, both]')
"""
@@ -1103,9 +1102,23 @@ def interval_range(
breaks: np.ndarray | TimedeltaIndex | DatetimeIndex
if is_number(endpoint):
+ dtype: np.dtype = np.dtype("int64")
if com.all_not_none(start, end, freq):
+ if (
+ isinstance(start, (float, np.float16))
+ or isinstance(end, (float, np.float16))
+ or isinstance(freq, (float, np.float16))
+ ):
+ dtype = np.dtype("float64")
+ elif (
+ isinstance(start, (np.integer, np.floating))
+ and isinstance(end, (np.integer, np.floating))
+ and start.dtype == end.dtype
+ ):
+ dtype = start.dtype
# 0.1 ensures we capture end
breaks = np.arange(start, end + (freq * 0.1), freq)
+ breaks = maybe_downcast_numeric(breaks, dtype)
else:
# compute the period/start/end if unspecified (at most one)
if periods is None:
@@ -1124,7 +1137,7 @@ def interval_range(
# expected "ndarray[Any, Any]" [
breaks = maybe_downcast_numeric(
breaks, # type: ignore[arg-type]
- np.dtype("int64"),
+ dtype,
)
else:
# delegate to the appropriate range function
@@ -1133,4 +1146,9 @@ def interval_range(
else:
breaks = timedelta_range(start=start, end=end, periods=periods, freq=freq)
- return IntervalIndex.from_breaks(breaks, name=name, closed=closed)
+ return IntervalIndex.from_breaks(
+ breaks,
+ name=name,
+ closed=closed,
+ dtype=IntervalDtype(subtype=breaks.dtype, closed=closed),
+ )
diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py
index 1352238eb60ec..2e554bc848ffe 100644
--- a/pandas/core/indexes/multi.py
+++ b/pandas/core/indexes/multi.py
@@ -37,6 +37,7 @@
F,
IgnoreRaise,
IndexLabel,
+ IndexT,
Scalar,
Self,
Shape,
@@ -100,17 +101,13 @@
ensure_index,
get_unanimous_names,
)
-from pandas.core.indexes.frozen import FrozenList
from pandas.core.ops.invalid import make_invalid_op
from pandas.core.sorting import (
get_group_index,
lexsort_indexer,
)
-from pandas.io.formats.printing import (
- get_adjustment,
- pprint_thing,
-)
+from pandas.io.formats.printing import pprint_thing
if TYPE_CHECKING:
from pandas import (
@@ -296,13 +293,13 @@ class MultiIndex(Index):
methods :meth:`MultiIndex.from_arrays`, :meth:`MultiIndex.from_product`
and :meth:`MultiIndex.from_tuples`. For example (using ``.from_arrays``):
- >>> arrays = [[1, 1, 2, 2], ['red', 'blue', 'red', 'blue']]
- >>> pd.MultiIndex.from_arrays(arrays, names=('number', 'color'))
+ >>> arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
+ >>> pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
MultiIndex([(1, 'red'),
(1, 'blue'),
(2, 'red'),
(2, 'blue')],
- names=['number', 'color'])
+ names=('number', 'color'))
See further examples for how to construct a MultiIndex in the doc strings
of the mentioned helper methods.
@@ -312,9 +309,9 @@ class MultiIndex(Index):
# initialize to zero-length tuples to make everything work
_typ = "multiindex"
- _names: list[Hashable | None] = []
- _levels = FrozenList()
- _codes = FrozenList()
+ _names: tuple[Hashable | None, ...] = ()
+ _levels: tuple[Index, ...] = ()
+ _codes: tuple[np.ndarray, ...] = ()
_comparables = ["names"]
sortorder: int | None
@@ -350,7 +347,7 @@ def __new__(
result._set_levels(levels, copy=copy, validate=False)
result._set_codes(codes, copy=copy, validate=False)
- result._names = [None] * len(levels)
+ result._names = (None,) * len(levels)
if names is not None:
# handles name validation
result._set_names(names)
@@ -369,15 +366,15 @@ def __new__(
return result
- def _validate_codes(self, level: list, code: list):
+ def _validate_codes(self, level: Index, code: np.ndarray) -> np.ndarray:
"""
Reassign code values as -1 if their corresponding levels are NaN.
Parameters
----------
- code : list
+ code : Index
Code to reassign.
- level : list
+ level : np.ndarray
Level to check for missing values (NaN, NaT, None).
Returns
@@ -387,24 +384,21 @@ def _validate_codes(self, level: list, code: list):
"""
null_mask = isna(level)
if np.any(null_mask):
- # error: Incompatible types in assignment
- # (expression has type "ndarray[Any, dtype[Any]]",
- # variable has type "List[Any]")
- code = np.where(null_mask[code], -1, code) # type: ignore[assignment]
+ code = np.where(null_mask[code], -1, code)
return code
def _verify_integrity(
self,
- codes: list | None = None,
- levels: list | None = None,
+ codes: tuple | None = None,
+ levels: tuple | None = None,
levels_to_verify: list[int] | range | None = None,
- ):
+ ) -> tuple:
"""
Parameters
----------
- codes : optional list
+ codes : optional tuple
Codes to check for validity. Defaults to current codes.
- levels : optional list
+ levels : optional tuple
Levels to check for validity. Defaults to current levels.
levels_to_validate: optional list
Specifies the levels to verify.
@@ -468,7 +462,7 @@ def _verify_integrity(
else:
result_codes.append(codes[i])
- new_codes = FrozenList(result_codes)
+ new_codes = tuple(result_codes)
return new_codes
@classmethod
@@ -505,13 +499,13 @@ def from_arrays(
Examples
--------
- >>> arrays = [[1, 1, 2, 2], ['red', 'blue', 'red', 'blue']]
- >>> pd.MultiIndex.from_arrays(arrays, names=('number', 'color'))
+ >>> arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
+ >>> pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
MultiIndex([(1, 'red'),
(1, 'blue'),
(2, 'red'),
(2, 'blue')],
- names=['number', 'color'])
+ names=('number', 'color'))
"""
error_msg = "Input must be a list / sequence of array-likes."
if not is_list_like(arrays):
@@ -576,14 +570,13 @@ def from_tuples(
Examples
--------
- >>> tuples = [(1, 'red'), (1, 'blue'),
- ... (2, 'red'), (2, 'blue')]
- >>> pd.MultiIndex.from_tuples(tuples, names=('number', 'color'))
+ >>> tuples = [(1, "red"), (1, "blue"), (2, "red"), (2, "blue")]
+ >>> pd.MultiIndex.from_tuples(tuples, names=("number", "color"))
MultiIndex([(1, 'red'),
(1, 'blue'),
(2, 'red'),
(2, 'blue')],
- names=['number', 'color'])
+ names=('number', 'color'))
"""
if not is_list_like(tuples):
raise TypeError("Input must be a list / sequence of tuple-likes.")
@@ -658,16 +651,15 @@ def from_product(
Examples
--------
>>> numbers = [0, 1, 2]
- >>> colors = ['green', 'purple']
- >>> pd.MultiIndex.from_product([numbers, colors],
- ... names=['number', 'color'])
+ >>> colors = ["green", "purple"]
+ >>> pd.MultiIndex.from_product([numbers, colors], names=["number", "color"])
MultiIndex([(0, 'green'),
(0, 'purple'),
(1, 'green'),
(1, 'purple'),
(2, 'green'),
(2, 'purple')],
- names=['number', 'color'])
+ names=('number', 'color'))
"""
from pandas.core.reshape.util import cartesian_product
@@ -720,9 +712,10 @@ def from_frame(
Examples
--------
- >>> df = pd.DataFrame([['HI', 'Temp'], ['HI', 'Precip'],
- ... ['NJ', 'Temp'], ['NJ', 'Precip']],
- ... columns=['a', 'b'])
+ >>> df = pd.DataFrame(
+ ... [["HI", "Temp"], ["HI", "Precip"], ["NJ", "Temp"], ["NJ", "Precip"]],
+ ... columns=["a", "b"],
+ ... )
>>> df
a b
0 HI Temp
@@ -735,16 +728,16 @@ def from_frame(
('HI', 'Precip'),
('NJ', 'Temp'),
('NJ', 'Precip')],
- names=['a', 'b'])
+ names=('a', 'b'))
Using explicit names, instead of the column names
- >>> pd.MultiIndex.from_frame(df, names=['state', 'observation'])
+ >>> pd.MultiIndex.from_frame(df, names=["state", "observation"])
MultiIndex([('HI', 'Temp'),
('HI', 'Precip'),
('NJ', 'Temp'),
('NJ', 'Precip')],
- names=['state', 'observation'])
+ names=('state', 'observation'))
"""
if not isinstance(df, ABCDataFrame):
raise TypeError("Input must be a DataFrame")
@@ -767,16 +760,18 @@ def _values(self) -> np.ndarray:
vals = index
if isinstance(vals.dtype, CategoricalDtype):
vals = cast("CategoricalIndex", vals)
- vals = vals._data._internal_get_values()
+ # Incompatible types in assignment (expression has type
+ # "ExtensionArray | ndarray[Any, Any]", variable has type "Index")
+ vals = vals._data._internal_get_values() # type: ignore[assignment]
if isinstance(vals.dtype, ExtensionDtype) or lib.is_np_dtype(
vals.dtype, "mM"
):
vals = vals.astype(object)
- vals = np.array(vals, copy=False)
- vals = algos.take_nd(vals, codes, fill_value=index._na_value)
- values.append(vals)
+ array_vals = np.asarray(vals)
+ array_vals = algos.take_nd(array_vals, codes, fill_value=index._na_value)
+ values.append(array_vals)
arr = lib.fast_zip(values)
return arr
@@ -807,8 +802,9 @@ def dtypes(self) -> Series:
Examples
--------
- >>> idx = pd.MultiIndex.from_product([(0, 1, 2), ('green', 'purple')],
- ... names=['number', 'color'])
+ >>> idx = pd.MultiIndex.from_product(
+ ... [(0, 1, 2), ("green", "purple")], names=["number", "color"]
+ ... )
>>> idx
MultiIndex([(0, 'green'),
(0, 'purple'),
@@ -816,7 +812,7 @@ def dtypes(self) -> Series:
(1, 'purple'),
(2, 'green'),
(2, 'purple')],
- names=['number', 'color'])
+ names=('number', 'color'))
>>> idx.dtypes
number int64
color object
@@ -842,7 +838,7 @@ def size(self) -> int:
# Levels Methods
@cache_readonly
- def levels(self) -> FrozenList:
+ def levels(self) -> tuple[Index, ...]:
"""
Levels of the MultiIndex.
@@ -861,10 +857,11 @@ def levels(self) -> FrozenList:
Examples
--------
- >>> index = pd.MultiIndex.from_product([['mammal'],
- ... ('goat', 'human', 'cat', 'dog')],
- ... names=['Category', 'Animals'])
- >>> leg_num = pd.DataFrame(data=(4, 2, 4, 4), index=index, columns=['Legs'])
+ >>> index = pd.MultiIndex.from_product(
+ ... [["mammal"], ("goat", "human", "cat", "dog")],
+ ... names=["Category", "Animals"],
+ ... )
+ >>> leg_num = pd.DataFrame(data=(4, 2, 4, 4), index=index, columns=["Legs"])
>>> leg_num
Legs
Category Animals
@@ -874,7 +871,8 @@ def levels(self) -> FrozenList:
dog 4
>>> leg_num.index.levels
- FrozenList([['mammal'], ['cat', 'dog', 'goat', 'human']])
+ (Index(['mammal'], dtype='object', name='Category'),
+ Index(['cat', 'dog', 'goat', 'human'], dtype='object', name='Animals'))
MultiIndex levels will not change even if the DataFrame using the MultiIndex
does not contain all them anymore.
@@ -889,7 +887,8 @@ def levels(self) -> FrozenList:
dog 4
>>> large_leg_num.index.levels
- FrozenList([['mammal'], ['cat', 'dog', 'goat', 'human']])
+ (Index(['mammal'], dtype='object', name='Category'),
+ Index(['cat', 'dog', 'goat', 'human'], dtype='object', name='Animals'))
"""
# Use cache_readonly to ensure that self.get_locs doesn't repeatedly
# create new IndexEngine
@@ -898,7 +897,7 @@ def levels(self) -> FrozenList:
for level in result:
# disallow midx.levels[0].name = "foo"
level._no_setting_name = True
- return FrozenList(result)
+ return tuple(result)
def _set_levels(
self,
@@ -921,16 +920,14 @@ def _set_levels(
raise ValueError("Length of levels must match length of level.")
if level is None:
- new_levels = FrozenList(
- ensure_index(lev, copy=copy)._view() for lev in levels
- )
- level_numbers = list(range(len(new_levels)))
+ new_levels = tuple(ensure_index(lev, copy=copy)._view() for lev in levels)
+ level_numbers: range | list[int] = range(len(new_levels))
else:
level_numbers = [self._get_level_number(lev) for lev in level]
new_levels_list = list(self._levels)
for lev_num, lev in zip(level_numbers, levels):
new_levels_list[lev_num] = ensure_index(lev, copy=copy)._view()
- new_levels = FrozenList(new_levels_list)
+ new_levels = tuple(new_levels_list)
if verify_integrity:
new_codes = self._verify_integrity(
@@ -939,7 +936,7 @@ def _set_levels(
self._codes = new_codes
names = self.names
- self._levels = new_levels
+ self._levels: tuple[Index, ...] = new_levels
if any(names):
self._set_names(names)
@@ -973,9 +970,9 @@ def set_levels(
... (2, "one"),
... (2, "two"),
... (3, "one"),
- ... (3, "two")
+ ... (3, "two"),
... ],
- ... names=["foo", "bar"]
+ ... names=["foo", "bar"],
... )
>>> idx
MultiIndex([(1, 'one'),
@@ -984,49 +981,49 @@ def set_levels(
(2, 'two'),
(3, 'one'),
(3, 'two')],
- names=['foo', 'bar'])
+ names=('foo', 'bar'))
- >>> idx.set_levels([['a', 'b', 'c'], [1, 2]])
+ >>> idx.set_levels([["a", "b", "c"], [1, 2]])
MultiIndex([('a', 1),
('a', 2),
('b', 1),
('b', 2),
('c', 1),
('c', 2)],
- names=['foo', 'bar'])
- >>> idx.set_levels(['a', 'b', 'c'], level=0)
+ names=('foo', 'bar'))
+ >>> idx.set_levels(["a", "b", "c"], level=0)
MultiIndex([('a', 'one'),
('a', 'two'),
('b', 'one'),
('b', 'two'),
('c', 'one'),
('c', 'two')],
- names=['foo', 'bar'])
- >>> idx.set_levels(['a', 'b'], level='bar')
+ names=('foo', 'bar'))
+ >>> idx.set_levels(["a", "b"], level="bar")
MultiIndex([(1, 'a'),
(1, 'b'),
(2, 'a'),
(2, 'b'),
(3, 'a'),
(3, 'b')],
- names=['foo', 'bar'])
+ names=('foo', 'bar'))
If any of the levels passed to ``set_levels()`` exceeds the
existing length, all of the values from that argument will
be stored in the MultiIndex levels, though the values will
be truncated in the MultiIndex output.
- >>> idx.set_levels([['a', 'b', 'c'], [1, 2, 3, 4]], level=[0, 1])
+ >>> idx.set_levels([["a", "b", "c"], [1, 2, 3, 4]], level=[0, 1])
MultiIndex([('a', 1),
('a', 2),
('b', 1),
('b', 2),
('c', 1),
('c', 2)],
- names=['foo', 'bar'])
- >>> idx.set_levels([['a', 'b', 'c'], [1, 2, 3, 4]], level=[0, 1]).levels
- FrozenList([['a', 'b', 'c'], [1, 2, 3, 4]])
- """
+ names=('foo', 'bar'))
+ >>> idx.set_levels([["a", "b", "c"], [1, 2, 3, 4]], level=[0, 1]).levels
+ (Index(['a', 'b', 'c'], dtype='object', name='foo'), Index([1, 2, 3, 4], dtype='int64', name='bar'))
+ """ # noqa: E501
if isinstance(levels, Index):
pass
@@ -1050,7 +1047,7 @@ def nlevels(self) -> int:
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([['a'], ['b'], ['c']])
+ >>> mi = pd.MultiIndex.from_arrays([["a"], ["b"], ["c"]])
>>> mi
MultiIndex([('a', 'b', 'c')],
)
@@ -1066,7 +1063,7 @@ def levshape(self) -> Shape:
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([['a'], ['b'], ['c']])
+ >>> mi = pd.MultiIndex.from_arrays([["a"], ["b"], ["c"]])
>>> mi
MultiIndex([('a', 'b', 'c')],
)
@@ -1079,7 +1076,30 @@ def levshape(self) -> Shape:
# Codes Methods
@property
- def codes(self) -> FrozenList:
+ def codes(self) -> tuple:
+ """
+ Codes of the MultiIndex.
+
+ Codes are the position of the index value in the list of level values
+ for each level.
+
+ Returns
+ -------
+ tuple of numpy.ndarray
+ The codes of the MultiIndex. Each array in the tuple corresponds
+ to a level in the MultiIndex.
+
+ See Also
+ --------
+ MultiIndex.set_codes : Set new codes on MultiIndex.
+
+ Examples
+ --------
+ >>> arrays = [[1, 1, 2, 2], ["red", "blue", "red", "blue"]]
+ >>> mi = pd.MultiIndex.from_arrays(arrays, names=("number", "color"))
+ >>> mi.codes
+ (array([0, 0, 1, 1], dtype=int8), array([1, 0, 1, 0], dtype=int8))
+ """
return self._codes
def _set_codes(
@@ -1099,7 +1119,7 @@ def _set_codes(
level_numbers: list[int] | range
if level is None:
- new_codes = FrozenList(
+ new_codes = tuple(
_coerce_indexer_frozen(level_codes, lev, copy=copy).view()
for lev, level_codes in zip(self._levels, codes)
)
@@ -1112,7 +1132,7 @@ def _set_codes(
new_codes_list[lev_num] = _coerce_indexer_frozen(
level_codes, lev, copy=copy
)
- new_codes = FrozenList(new_codes_list)
+ new_codes = tuple(new_codes_list)
if verify_integrity:
new_codes = self._verify_integrity(
@@ -1153,32 +1173,32 @@ def set_codes(
(1, 'two'),
(2, 'one'),
(2, 'two')],
- names=['foo', 'bar'])
+ names=('foo', 'bar'))
>>> idx.set_codes([[1, 0, 1, 0], [0, 0, 1, 1]])
MultiIndex([(2, 'one'),
(1, 'one'),
(2, 'two'),
(1, 'two')],
- names=['foo', 'bar'])
+ names=('foo', 'bar'))
>>> idx.set_codes([1, 0, 1, 0], level=0)
MultiIndex([(2, 'one'),
(1, 'two'),
(2, 'one'),
(1, 'two')],
- names=['foo', 'bar'])
- >>> idx.set_codes([0, 0, 1, 1], level='bar')
+ names=('foo', 'bar'))
+ >>> idx.set_codes([0, 0, 1, 1], level="bar")
MultiIndex([(1, 'one'),
(1, 'one'),
(2, 'two'),
(2, 'two')],
- names=['foo', 'bar'])
+ names=('foo', 'bar'))
>>> idx.set_codes([[1, 0, 1, 0], [0, 0, 1, 1]], level=[0, 1])
MultiIndex([(2, 'one'),
(1, 'one'),
(2, 'two'),
(1, 'two')],
- names=['foo', 'bar'])
+ names=('foo', 'bar'))
"""
level, codes = _require_listlike(level, codes, "Codes")
@@ -1275,7 +1295,7 @@ def copy( # type: ignore[override]
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([['a'], ['b'], ['c']])
+ >>> mi = pd.MultiIndex.from_arrays([["a"], ["b"], ["c"]])
>>> mi
MultiIndex([('a', 'b', 'c')],
)
@@ -1309,7 +1329,7 @@ def copy( # type: ignore[override]
new_index._id = self._id
return new_index
- def __array__(self, dtype=None) -> np.ndarray:
+ def __array__(self, dtype=None, copy=None) -> np.ndarray:
"""the array interface, return my values"""
return self.values
@@ -1421,87 +1441,6 @@ def _get_values_for_csv(
)
return mi._values
- def format(
- self,
- name: bool | None = None,
- formatter: Callable | None = None,
- na_rep: str | None = None,
- names: bool = False,
- space: int = 2,
- sparsify=None,
- adjoin: bool = True,
- ) -> list:
- warnings.warn(
- # GH#55413
- f"{type(self).__name__}.format is deprecated and will be removed "
- "in a future version. Convert using index.astype(str) or "
- "index.map(formatter) instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- if name is not None:
- names = name
-
- if len(self) == 0:
- return []
-
- stringified_levels = []
- for lev, level_codes in zip(self.levels, self.codes):
- na = na_rep if na_rep is not None else _get_na_rep(lev.dtype)
-
- if len(lev) > 0:
- formatted = lev.take(level_codes).format(formatter=formatter)
-
- # we have some NA
- mask = level_codes == -1
- if mask.any():
- formatted = np.array(formatted, dtype=object)
- formatted[mask] = na
- formatted = formatted.tolist()
-
- else:
- # weird all NA case
- formatted = [
- pprint_thing(na if isna(x) else x, escape_chars=("\t", "\r", "\n"))
- for x in algos.take_nd(lev._values, level_codes)
- ]
- stringified_levels.append(formatted)
-
- result_levels = []
- for lev, lev_name in zip(stringified_levels, self.names):
- level = []
-
- if names:
- level.append(
- pprint_thing(lev_name, escape_chars=("\t", "\r", "\n"))
- if lev_name is not None
- else ""
- )
-
- level.extend(np.array(lev, dtype=object))
- result_levels.append(level)
-
- if sparsify is None:
- sparsify = get_option("display.multi_sparse")
-
- if sparsify:
- sentinel: Literal[""] | bool | lib.NoDefault = ""
- # GH3547 use value of sparsify as sentinel if it's "Falsey"
- assert isinstance(sparsify, bool) or sparsify is lib.no_default
- if sparsify in [False, lib.no_default]:
- sentinel = sparsify
- # little bit of a kludge job for #1217
- result_levels = sparsify_labels(
- result_levels, start=int(names), sentinel=sentinel
- )
-
- if adjoin:
- adj = get_adjustment()
- return adj.adjoin(space, *result_levels).split("\n")
- else:
- return result_levels
-
def _format_multi(
self,
*,
@@ -1512,6 +1451,7 @@ def _format_multi(
if len(self) == 0:
return []
+ formatted: Iterable
stringified_levels = []
for lev, level_codes in zip(self.levels, self.codes):
na = _get_na_rep(lev.dtype)
@@ -1536,7 +1476,9 @@ def _format_multi(
stringified_levels.append(formatted)
result_levels = []
- for lev, lev_name in zip(stringified_levels, self.names):
+ # Incompatible types in assignment (expression has type "Iterable[Any]",
+ # variable has type "Index")
+ for lev, lev_name in zip(stringified_levels, self.names): # type: ignore[assignment]
level = []
if include_names:
@@ -1568,8 +1510,8 @@ def _format_multi(
# --------------------------------------------------------------------
# Names Methods
- def _get_names(self) -> FrozenList:
- return FrozenList(self._names)
+ def _get_names(self) -> tuple[Hashable | None, ...]:
+ return self._names
def _set_names(self, names, *, level=None, validate: bool = True) -> None:
"""
@@ -1616,6 +1558,7 @@ def _set_names(self, names, *, level=None, validate: bool = True) -> None:
level = [self._get_level_number(lev) for lev in level]
# set the name
+ new_names = list(self._names)
for lev, name in zip(level, names):
if name is not None:
# GH 20527
@@ -1624,7 +1567,8 @@ def _set_names(self, names, *, level=None, validate: bool = True) -> None:
raise TypeError(
f"{type(self).__name__}.name must be a hashable type"
)
- self._names[lev] = name
+ new_names[lev] = name
+ self._names = tuple(new_names)
# If .levels has been accessed, the names in our cache will be stale.
self._reset_cache()
@@ -1638,13 +1582,14 @@ def _set_names(self, names, *, level=None, validate: bool = True) -> None:
Examples
--------
>>> mi = pd.MultiIndex.from_arrays(
- ... [[1, 2], [3, 4], [5, 6]], names=['x', 'y', 'z'])
+ ... [[1, 2], [3, 4], [5, 6]], names=['x', 'y', 'z']
+ ... )
>>> mi
MultiIndex([(1, 3, 5),
(2, 4, 6)],
- names=['x', 'y', 'z'])
+ names=('x', 'y', 'z'))
>>> mi.names
- FrozenList(['x', 'y', 'z'])
+ ('x', 'y', 'z')
""",
)
@@ -1809,14 +1754,14 @@ def get_level_values(self, level) -> Index: # type: ignore[override]
--------
Create a MultiIndex:
- >>> mi = pd.MultiIndex.from_arrays((list('abc'), list('def')))
- >>> mi.names = ['level_1', 'level_2']
+ >>> mi = pd.MultiIndex.from_arrays((list("abc"), list("def")))
+ >>> mi.names = ["level_1", "level_2"]
Get level values by supplying level as either integer or name:
>>> mi.get_level_values(0)
Index(['a', 'b', 'c'], dtype='object', name='level_1')
- >>> mi.get_level_values('level_2')
+ >>> mi.get_level_values("level_2")
Index(['d', 'e', 'f'], dtype='object', name='level_2')
If a level contains missing values, the return type of the level
@@ -1877,7 +1822,7 @@ def to_frame(
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([['a', 'b'], ['c', 'd']])
+ >>> mi = pd.MultiIndex.from_arrays([["a", "b"], ["c", "d"]])
>>> mi
MultiIndex([('a', 'c'),
('b', 'd')],
@@ -1895,7 +1840,7 @@ def to_frame(
0 a c
1 b d
- >>> df = mi.to_frame(name=['x', 'y'])
+ >>> df = mi.to_frame(name=["x", "y"])
>>> df
x y
a c a c
@@ -1954,8 +1899,8 @@ def to_flat_index(self) -> Index: # type: ignore[override]
Examples
--------
>>> index = pd.MultiIndex.from_product(
- ... [['foo', 'bar'], ['baz', 'qux']],
- ... names=['a', 'b'])
+ ... [["foo", "bar"], ["baz", "qux"]], names=["a", "b"]
+ ... )
>>> index.to_flat_index()
Index([('foo', 'baz'), ('foo', 'qux'),
('bar', 'baz'), ('bar', 'qux')],
@@ -1976,25 +1921,29 @@ def _is_lexsorted(self) -> bool:
In the below examples, the first level of the MultiIndex is sorted because
a>> pd.MultiIndex.from_arrays([['a', 'b', 'c'],
- ... ['d', 'e', 'f']])._is_lexsorted()
+ >>> pd.MultiIndex.from_arrays(
+ ... [["a", "b", "c"], ["d", "e", "f"]]
+ ... )._is_lexsorted()
True
- >>> pd.MultiIndex.from_arrays([['a', 'b', 'c'],
- ... ['d', 'f', 'e']])._is_lexsorted()
+ >>> pd.MultiIndex.from_arrays(
+ ... [["a", "b", "c"], ["d", "f", "e"]]
+ ... )._is_lexsorted()
True
In case there is a tie, the lexicographical sorting looks
at the next level of the MultiIndex.
- >>> pd.MultiIndex.from_arrays([[0, 1, 1], ['a', 'b', 'c']])._is_lexsorted()
+ >>> pd.MultiIndex.from_arrays([[0, 1, 1], ["a", "b", "c"]])._is_lexsorted()
True
- >>> pd.MultiIndex.from_arrays([[0, 1, 1], ['a', 'c', 'b']])._is_lexsorted()
+ >>> pd.MultiIndex.from_arrays([[0, 1, 1], ["a", "c", "b"]])._is_lexsorted()
False
- >>> pd.MultiIndex.from_arrays([['a', 'a', 'b', 'b'],
- ... ['aa', 'bb', 'aa', 'bb']])._is_lexsorted()
+ >>> pd.MultiIndex.from_arrays(
+ ... [["a", "a", "b", "b"], ["aa", "bb", "aa", "bb"]]
+ ... )._is_lexsorted()
True
- >>> pd.MultiIndex.from_arrays([['a', 'a', 'b', 'b'],
- ... ['bb', 'aa', 'aa', 'bb']])._is_lexsorted()
+ >>> pd.MultiIndex.from_arrays(
+ ... [["a", "a", "b", "b"], ["bb", "aa", "aa", "bb"]]
+ ... )._is_lexsorted()
False
"""
return self._lexsort_depth == self.nlevels
@@ -2031,8 +1980,9 @@ def _sort_levels_monotonic(self, raise_if_incomparable: bool = False) -> MultiIn
Examples
--------
- >>> mi = pd.MultiIndex(levels=[['a', 'b'], ['bb', 'aa']],
- ... codes=[[0, 0, 1, 1], [0, 1, 0, 1]])
+ >>> mi = pd.MultiIndex(
+ ... levels=[["a", "b"], ["bb", "aa"]], codes=[[0, 0, 1, 1], [0, 1, 0, 1]]
+ ... )
>>> mi
MultiIndex([('a', 'bb'),
('a', 'aa'),
@@ -2095,7 +2045,7 @@ def remove_unused_levels(self) -> MultiIndex:
Examples
--------
- >>> mi = pd.MultiIndex.from_product([range(2), list('ab')])
+ >>> mi = pd.MultiIndex.from_product([range(2), list("ab")])
>>> mi
MultiIndex([(0, 'a'),
(0, 'b'),
@@ -2113,7 +2063,7 @@ def remove_unused_levels(self) -> MultiIndex:
>>> mi2 = mi[2:].remove_unused_levels()
>>> mi2.levels
- FrozenList([[1], ['a', 'b']])
+ (RangeIndex(start=1, stop=2, step=1), Index(['a', 'b'], dtype='object'))
"""
new_levels = []
new_codes = []
@@ -2247,6 +2197,9 @@ def take(
# only fill if we are passing a non-None fill_value
allow_fill = self._maybe_disallow_fill(allow_fill, fill_value, indices)
+ if indices.ndim == 1 and lib.is_range_indexer(indices, len(self)):
+ return self.copy()
+
na_value = -1
taken = [lab.take(indices) for lab in self.codes]
@@ -2279,7 +2232,7 @@ def append(self, other):
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([['a'], ['b']])
+ >>> mi = pd.MultiIndex.from_arrays([["a"], ["b"]])
>>> mi
MultiIndex([('a', 'b')],
)
@@ -2374,8 +2327,9 @@ def drop( # type: ignore[override]
Examples
--------
- >>> idx = pd.MultiIndex.from_product([(0, 1, 2), ('green', 'purple')],
- ... names=["number", "color"])
+ >>> idx = pd.MultiIndex.from_product(
+ ... [(0, 1, 2), ("green", "purple")], names=["number", "color"]
+ ... )
>>> idx
MultiIndex([(0, 'green'),
(0, 'purple'),
@@ -2383,26 +2337,26 @@ def drop( # type: ignore[override]
(1, 'purple'),
(2, 'green'),
(2, 'purple')],
- names=['number', 'color'])
- >>> idx.drop([(1, 'green'), (2, 'purple')])
+ names=('number', 'color'))
+ >>> idx.drop([(1, "green"), (2, "purple")])
MultiIndex([(0, 'green'),
(0, 'purple'),
(1, 'purple'),
(2, 'green')],
- names=['number', 'color'])
+ names=('number', 'color'))
We can also drop from a specific level.
- >>> idx.drop('green', level='color')
+ >>> idx.drop("green", level="color")
MultiIndex([(0, 'purple'),
(1, 'purple'),
(2, 'purple')],
- names=['number', 'color'])
+ names=('number', 'color'))
>>> idx.drop([1, 2], level=0)
MultiIndex([(0, 'green'),
(0, 'purple')],
- names=['number', 'color'])
+ names=('number', 'color'))
"""
if level is not None:
return self._drop_from_level(codes, level, errors)
@@ -2425,7 +2379,7 @@ def drop( # type: ignore[override]
step = loc.step if loc.step is not None else 1
inds.extend(range(loc.start, loc.stop, step))
elif com.is_bool_indexer(loc):
- if self._lexsort_depth == 0:
+ if get_option("performance_warnings") and self._lexsort_depth == 0:
warnings.warn(
"dropping on a non-lexsorted multi-index "
"without a level parameter may impact performance.",
@@ -2492,8 +2446,9 @@ def swaplevel(self, i=-2, j=-1) -> MultiIndex:
Examples
--------
- >>> mi = pd.MultiIndex(levels=[['a', 'b'], ['bb', 'aa']],
- ... codes=[[0, 0, 1, 1], [0, 1, 0, 1]])
+ >>> mi = pd.MultiIndex(
+ ... levels=[["a", "b"], ["bb", "aa"]], codes=[[0, 0, 1, 1], [0, 1, 0, 1]]
+ ... )
>>> mi
MultiIndex([('a', 'bb'),
('a', 'aa'),
@@ -2538,21 +2493,21 @@ def reorder_levels(self, order) -> MultiIndex:
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([[1, 2], [3, 4]], names=['x', 'y'])
+ >>> mi = pd.MultiIndex.from_arrays([[1, 2], [3, 4]], names=["x", "y"])
>>> mi
MultiIndex([(1, 3),
(2, 4)],
- names=['x', 'y'])
+ names=('x', 'y'))
>>> mi.reorder_levels(order=[1, 0])
MultiIndex([(3, 1),
(4, 2)],
- names=['y', 'x'])
+ names=('y', 'x'))
- >>> mi.reorder_levels(order=['y', 'x'])
+ >>> mi.reorder_levels(order=["y", "x"])
MultiIndex([(3, 1),
(4, 2)],
- names=['y', 'x'])
+ names=('y', 'x'))
"""
order = [self._get_level_number(i) for i in order]
result = self._reorder_ilevels(order)
@@ -2673,7 +2628,8 @@ def sortlevel(
# error: Item "Hashable" of "Union[Hashable, Sequence[Hashable]]" has
# no attribute "__iter__" (not iterable)
level = [
- self._get_level_number(lev) for lev in level # type: ignore[union-attr]
+ self._get_level_number(lev)
+ for lev in level # type: ignore[union-attr]
]
sortorder = None
@@ -2722,7 +2678,7 @@ def _wrap_reindex_result(self, target, indexer, preserve_names: bool):
target = self._maybe_preserve_names(target, preserve_names)
return target
- def _maybe_preserve_names(self, target: Index, preserve_names: bool) -> Index:
+ def _maybe_preserve_names(self, target: IndexT, preserve_names: bool) -> IndexT:
if (
preserve_names
and target.nlevels == self.nlevels
@@ -2823,18 +2779,18 @@ def get_slice_bound(
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([list('abbc'), list('gefd')])
+ >>> mi = pd.MultiIndex.from_arrays([list("abbc"), list("gefd")])
Get the locations from the leftmost 'b' in the first level
until the end of the multiindex:
- >>> mi.get_slice_bound('b', side="left")
+ >>> mi.get_slice_bound("b", side="left")
1
Like above, but if you get the locations from the rightmost
'b' in the first level and 'f' in the second level:
- >>> mi.get_slice_bound(('b','f'), side="right")
+ >>> mi.get_slice_bound(("b", "f"), side="right")
3
See Also
@@ -2878,19 +2834,20 @@ def slice_locs(self, start=None, end=None, step=None) -> tuple[int, int]:
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([list('abbd'), list('deff')],
- ... names=['A', 'B'])
+ >>> mi = pd.MultiIndex.from_arrays(
+ ... [list("abbd"), list("deff")], names=["A", "B"]
+ ... )
Get the slice locations from the beginning of 'b' in the first level
until the end of the multiindex:
- >>> mi.slice_locs(start='b')
+ >>> mi.slice_locs(start="b")
(1, 4)
Like above, but stop at the end of 'b' in the first level and 'f' in
the second level:
- >>> mi.slice_locs(start='b', end=('b', 'f'))
+ >>> mi.slice_locs(start="b", end=("b", "f"))
(1, 3)
See Also
@@ -2920,7 +2877,9 @@ def _partial_tup_index(self, tup: tuple, side: Literal["left", "right"] = "left"
if lab not in lev and not isna(lab):
# short circuit
try:
- loc = algos.searchsorted(lev, lab, side=side)
+ # Argument 1 to "searchsorted" has incompatible type "Index";
+ # expected "ExtensionArray | ndarray[Any, Any]"
+ loc = algos.searchsorted(lev, lab, side=side) # type: ignore[arg-type]
except TypeError as err:
# non-comparable e.g. test_slice_locs_with_type_mismatch
raise TypeError(f"Level type mismatch: {lab}") from err
@@ -3012,12 +2971,12 @@ def get_loc(self, key):
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([list('abb'), list('def')])
+ >>> mi = pd.MultiIndex.from_arrays([list("abb"), list("def")])
- >>> mi.get_loc('b')
+ >>> mi.get_loc("b")
slice(1, 3, None)
- >>> mi.get_loc(('b', 'e'))
+ >>> mi.get_loc(("b", "e"))
1
"""
self._check_indexing_error(key)
@@ -3055,7 +3014,7 @@ def _maybe_to_slice(loc):
raise KeyError(key) from err
except TypeError:
# e.g. test_partial_slicing_with_multiindex partial string slicing
- loc, _ = self.get_loc_level(key, list(range(self.nlevels)))
+ loc, _ = self.get_loc_level(key, range(self.nlevels))
return loc
# -- partial selection or non-unique index
@@ -3082,11 +3041,12 @@ def _maybe_to_slice(loc):
if not follow_key:
return slice(start, stop)
- warnings.warn(
- "indexing past lexsort depth may impact performance.",
- PerformanceWarning,
- stacklevel=find_stack_level(),
- )
+ if get_option("performance_warnings"):
+ warnings.warn(
+ "indexing past lexsort depth may impact performance.",
+ PerformanceWarning,
+ stacklevel=find_stack_level(),
+ )
loc = np.arange(start, stop, dtype=np.intp)
@@ -3130,19 +3090,18 @@ def get_loc_level(self, key, level: IndexLabel = 0, drop_level: bool = True):
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([list('abb'), list('def')],
- ... names=['A', 'B'])
+ >>> mi = pd.MultiIndex.from_arrays([list("abb"), list("def")], names=["A", "B"])
- >>> mi.get_loc_level('b')
+ >>> mi.get_loc_level("b")
(slice(1, 3, None), Index(['e', 'f'], dtype='object', name='B'))
- >>> mi.get_loc_level('e', level='B')
+ >>> mi.get_loc_level("e", level="B")
(array([False, True, False]), Index(['b'], dtype='object', name='A'))
- >>> mi.get_loc_level(['b', 'e'])
+ >>> mi.get_loc_level(["b", "e"])
(1, None)
"""
- if not isinstance(level, (list, tuple)):
+ if not isinstance(level, (range, list, tuple)):
level = self._get_level_number(level)
else:
level = [self._get_level_number(lev) for lev in level]
@@ -3397,7 +3356,7 @@ def convert_indexer(start, stop, step, indexer=indexer, codes=level_codes):
locs = (level_codes >= idx.start) & (level_codes < idx.stop)
return locs
- locs = np.array(level_codes == idx, dtype=bool, copy=False)
+ locs = np.asarray(level_codes == idx, dtype=bool)
if not locs.any():
# The label is present in self.levels[level] but unused:
@@ -3441,15 +3400,15 @@ def get_locs(self, seq) -> npt.NDArray[np.intp]:
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([list('abb'), list('def')])
+ >>> mi = pd.MultiIndex.from_arrays([list("abb"), list("def")])
- >>> mi.get_locs('b') # doctest: +SKIP
+ >>> mi.get_locs("b") # doctest: +SKIP
array([1, 2], dtype=int64)
- >>> mi.get_locs([slice(None), ['e', 'f']]) # doctest: +SKIP
+ >>> mi.get_locs([slice(None), ["e", "f"]]) # doctest: +SKIP
array([1, 2], dtype=int64)
- >>> mi.get_locs([[True, False, True], slice('e', 'f')]) # doctest: +SKIP
+ >>> mi.get_locs([[True, False, True], slice("e", "f")]) # doctest: +SKIP
array([2], dtype=int64)
"""
@@ -3487,6 +3446,8 @@ def _to_bool_indexer(indexer) -> npt.NDArray[np.bool_]:
"cannot index with a boolean indexer that "
"is not the same length as the index"
)
+ if isinstance(k, (ABCSeries, Index)):
+ k = k._values
lvl_indexer = np.asarray(k)
if indexer is None:
lvl_indexer = lvl_indexer.copy()
@@ -3586,7 +3547,7 @@ def _reorder_indexer(
k_codes = self.levels[i].get_indexer(k)
k_codes = k_codes[k_codes >= 0] # Filter absent keys
# True if the given codes are not ordered
- need_sort = (k_codes[:-1] > k_codes[1:]).any()
+ need_sort = bool((k_codes[:-1] > k_codes[1:]).any())
else:
need_sort = True
elif isinstance(k, slice):
@@ -3628,7 +3589,7 @@ def _reorder_indexer(
new_order = key_order_map[self.codes[i][indexer]]
elif isinstance(k, slice) and k.step is not None and k.step < 0:
# flip order for negative step
- new_order = np.arange(n)[::-1][indexer]
+ new_order = np.arange(n - 1, -1, -1)[indexer]
elif isinstance(k, slice) and k.start is None and k.stop is None:
# slice(None) should not determine order GH#31330
new_order = np.ones((n,), dtype=np.intp)[indexer]
@@ -3659,11 +3620,11 @@ def truncate(self, before=None, after=None) -> MultiIndex:
Examples
--------
- >>> mi = pd.MultiIndex.from_arrays([['a', 'b', 'c'], ['x', 'y', 'z']])
+ >>> mi = pd.MultiIndex.from_arrays([["a", "b", "c"], ["x", "y", "z"]])
>>> mi
MultiIndex([('a', 'x'), ('b', 'y'), ('c', 'z')],
)
- >>> mi.truncate(before='a', after='b')
+ >>> mi.truncate(before="a", after="b")
MultiIndex([('a', 'x'), ('b', 'y')],
)
"""
@@ -3721,31 +3682,16 @@ def equals(self, other: object) -> bool:
other_mask = other_codes == -1
if not np.array_equal(self_mask, other_mask):
return False
- self_codes = self_codes[~self_mask]
- self_values = self.levels[i]._values.take(self_codes)
-
- other_codes = other_codes[~other_mask]
- other_values = other.levels[i]._values.take(other_codes)
-
- # since we use NaT both datetime64 and timedelta64 we can have a
- # situation where a level is typed say timedelta64 in self (IOW it
- # has other values than NaT) but types datetime64 in other (where
- # its all NaT) but these are equivalent
- if len(self_values) == 0 and len(other_values) == 0:
- continue
-
- if not isinstance(self_values, np.ndarray):
- # i.e. ExtensionArray
- if not self_values.equals(other_values):
- return False
- elif not isinstance(other_values, np.ndarray):
- # i.e. other is ExtensionArray
- if not other_values.equals(self_values):
- return False
- else:
- if not array_equivalent(self_values, other_values):
- return False
-
+ self_level = self.levels[i]
+ other_level = other.levels[i]
+ new_codes = recode_for_categories(
+ other_codes, other_level, self_level, copy=False
+ )
+ if not np.array_equal(self_codes, new_codes):
+ return False
+ if not self_level[:0].equals(other_level[:0]):
+ # e.g. Int64 != int64
+ return False
return True
def equal_levels(self, other: MultiIndex) -> bool:
@@ -4034,7 +3980,7 @@ def isin(self, values, level=None) -> npt.NDArray[np.bool_]:
__invert__ = make_invalid_op("__invert__")
-def _lexsort_depth(codes: list[np.ndarray], nlevels: int) -> int:
+def _lexsort_depth(codes: tuple[np.ndarray], nlevels: int) -> int:
"""Count depth (up to a maximum of `nlevels`) with which codes are lexsorted."""
int64_codes = [ensure_int64(level_codes) for level_codes in codes]
for k in range(nlevels, 0, -1):
@@ -4056,8 +4002,6 @@ def sparsify_labels(label_list, start: int = 0, sentinel: object = ""):
for i, (p, t) in enumerate(zip(prev, cur)):
if i == k - 1:
sparse_cur.append(t)
- # error: Argument 1 to "append" of "list" has incompatible
- # type "list[Any]"; expected "tuple[Any, ...]"
result.append(sparse_cur) # type: ignore[arg-type]
break
@@ -4065,8 +4009,6 @@ def sparsify_labels(label_list, start: int = 0, sentinel: object = ""):
sparse_cur.append(sentinel)
else:
sparse_cur.extend(cur[i:])
- # error: Argument 1 to "append" of "list" has incompatible
- # type "list[Any]"; expected "tuple[Any, ...]"
result.append(sparse_cur) # type: ignore[arg-type]
break
diff --git a/pandas/core/indexes/period.py b/pandas/core/indexes/period.py
index b2f1933800fd3..edd1fdd4da943 100644
--- a/pandas/core/indexes/period.py
+++ b/pandas/core/indexes/period.py
@@ -5,7 +5,6 @@
timedelta,
)
from typing import TYPE_CHECKING
-import warnings
import numpy as np
@@ -22,7 +21,6 @@
cache_readonly,
doc,
)
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.common import is_integer
from pandas.core.dtypes.dtypes import PeriodDtype
@@ -94,39 +92,14 @@ class PeriodIndex(DatetimeIndexOpsMixin):
----------
data : array-like (1d int np.ndarray or PeriodArray), optional
Optional period-like data to construct index with.
- copy : bool
- Make a copy of input ndarray.
freq : str or period object, optional
One of pandas period strings or corresponding objects.
- year : int, array, or Series, default None
-
- .. deprecated:: 2.2.0
- Use PeriodIndex.from_fields instead.
- month : int, array, or Series, default None
-
- .. deprecated:: 2.2.0
- Use PeriodIndex.from_fields instead.
- quarter : int, array, or Series, default None
-
- .. deprecated:: 2.2.0
- Use PeriodIndex.from_fields instead.
- day : int, array, or Series, default None
-
- .. deprecated:: 2.2.0
- Use PeriodIndex.from_fields instead.
- hour : int, array, or Series, default None
-
- .. deprecated:: 2.2.0
- Use PeriodIndex.from_fields instead.
- minute : int, array, or Series, default None
-
- .. deprecated:: 2.2.0
- Use PeriodIndex.from_fields instead.
- second : int, array, or Series, default None
-
- .. deprecated:: 2.2.0
- Use PeriodIndex.from_fields instead.
dtype : str or PeriodDtype, default None
+ A dtype from which to extract a freq.
+ copy : bool
+ Make a copy of input ndarray.
+ name : str, default None
+ Name of the resulting PeriodIndex.
Attributes
----------
@@ -161,6 +134,12 @@ class PeriodIndex(DatetimeIndexOpsMixin):
from_fields
from_ordinals
+ Raises
+ ------
+ ValueError
+ Passing the parameter data as a list without specifying either freq or
+ dtype will raise a ValueError: "freq not specified and cannot be inferred"
+
See Also
--------
Index : The base pandas Index type.
@@ -171,7 +150,7 @@ class PeriodIndex(DatetimeIndexOpsMixin):
Examples
--------
- >>> idx = pd.PeriodIndex.from_fields(year=[2000, 2002], quarter=[1, 3])
+ >>> idx = pd.PeriodIndex(data=["2000Q1", "2002Q3"], freq="Q")
>>> idx
PeriodIndex(['2000Q1', '2002Q3'], dtype='period[Q-DEC]')
"""
@@ -200,7 +179,7 @@ def _resolution_obj(self) -> Resolution:
@doc(
PeriodArray.asfreq,
- other="pandas.arrays.PeriodArray",
+ other="arrays.PeriodArray",
other_name="PeriodArray",
**_shared_doc_kwargs,
)
@@ -234,84 +213,29 @@ def second(self) -> Index:
def __new__(
cls,
data=None,
- ordinal=None,
freq=None,
dtype: Dtype | None = None,
copy: bool = False,
name: Hashable | None = None,
- **fields,
) -> Self:
- valid_field_set = {
- "year",
- "month",
- "day",
- "quarter",
- "hour",
- "minute",
- "second",
- }
-
refs = None
if not copy and isinstance(data, (Index, ABCSeries)):
refs = data._references
- if not set(fields).issubset(valid_field_set):
- argument = next(iter(set(fields) - valid_field_set))
- raise TypeError(f"__new__() got an unexpected keyword argument {argument}")
- elif len(fields):
- # GH#55960
- warnings.warn(
- "Constructing PeriodIndex from fields is deprecated. Use "
- "PeriodIndex.from_fields instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- if ordinal is not None:
- # GH#55960
- warnings.warn(
- "The 'ordinal' keyword in PeriodIndex is deprecated and will "
- "be removed in a future version. Use PeriodIndex.from_ordinals "
- "instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
name = maybe_extract_name(name, data, cls)
- if data is None and ordinal is None:
- # range-based.
- if not fields:
- # test_pickle_compat_construction
- cls._raise_scalar_data_error(None)
- data = cls.from_fields(**fields, freq=freq)._data
- copy = False
+ freq = validate_dtype_freq(dtype, freq)
- elif fields:
- if data is not None:
- raise ValueError("Cannot pass both data and fields")
- raise ValueError("Cannot pass both ordinal and fields")
+ # PeriodIndex allow PeriodIndex(period_index, freq=different)
+ # Let's not encourage that kind of behavior in PeriodArray.
- else:
- freq = validate_dtype_freq(dtype, freq)
-
- # PeriodIndex allow PeriodIndex(period_index, freq=different)
- # Let's not encourage that kind of behavior in PeriodArray.
-
- if freq and isinstance(data, cls) and data.freq != freq:
- # TODO: We can do some of these with no-copy / coercion?
- # e.g. D -> 2D seems to be OK
- data = data.asfreq(freq)
-
- if data is None and ordinal is not None:
- ordinal = np.asarray(ordinal, dtype=np.int64)
- dtype = PeriodDtype(freq)
- data = PeriodArray(ordinal, dtype=dtype)
- elif data is not None and ordinal is not None:
- raise ValueError("Cannot pass both data and ordinal")
- else:
- # don't pass copy here, since we copy later.
- data = period_array(data=data, freq=freq)
+ if freq and isinstance(data, cls) and data.freq != freq:
+ # TODO: We can do some of these with no-copy / coercion?
+ # e.g. D -> 2D seems to be OK
+ data = data.asfreq(freq)
+
+ # don't pass copy here, since we copy later.
+ data = period_array(data=data, freq=freq)
if copy:
data = data.copy()
@@ -331,6 +255,31 @@ def from_fields(
second=None,
freq=None,
) -> Self:
+ """
+ Construct a PeriodIndex from fields (year, month, day, etc.).
+
+ Parameters
+ ----------
+ year : int, array, or Series, default None
+ quarter : int, array, or Series, default None
+ month : int, array, or Series, default None
+ day : int, array, or Series, default None
+ hour : int, array, or Series, default None
+ minute : int, array, or Series, default None
+ second : int, array, or Series, default None
+ freq : str or period object, optional
+ One of pandas period strings or corresponding objects.
+
+ Returns
+ -------
+ PeriodIndex
+
+ Examples
+ --------
+ >>> idx = pd.PeriodIndex.from_fields(year=[2000, 2002], quarter=[1, 3])
+ >>> idx
+ PeriodIndex(['2000Q1', '2002Q3'], dtype='period[Q-DEC]')
+ """
fields = {
"year": year,
"quarter": quarter,
@@ -346,6 +295,28 @@ def from_fields(
@classmethod
def from_ordinals(cls, ordinals, *, freq, name=None) -> Self:
+ """
+ Construct a PeriodIndex from ordinals.
+
+ Parameters
+ ----------
+ ordinals : array-like of int
+ The period offsets from the proleptic Gregorian epoch.
+ freq : str or period object
+ One of pandas period strings or corresponding objects.
+ name : str, default None
+ Name of the resulting PeriodIndex.
+
+ Returns
+ -------
+ PeriodIndex
+
+ Examples
+ --------
+ >>> idx = pd.PeriodIndex.from_ordinals([-1, 0, 1], freq="Q")
+ >>> idx
+ PeriodIndex(['1969Q4', '1970Q1', '1970Q2'], dtype='period[Q-DEC]')
+ """
ordinals = np.asarray(ordinals, dtype=np.int64)
dtype = PeriodDtype(freq)
data = PeriodArray._simple_new(ordinals, dtype=dtype)
@@ -580,12 +551,12 @@ def period_range(
Of the three parameters: ``start``, ``end``, and ``periods``, exactly two
must be specified.
- To learn more about the frequency strings, please see `this link
- `__.
+ To learn more about the frequency strings, please see
+ :ref:`this link`.
Examples
--------
- >>> pd.period_range(start='2017-01-01', end='2018-01-01', freq='M')
+ >>> pd.period_range(start="2017-01-01", end="2018-01-01", freq="M")
PeriodIndex(['2017-01', '2017-02', '2017-03', '2017-04', '2017-05', '2017-06',
'2017-07', '2017-08', '2017-09', '2017-10', '2017-11', '2017-12',
'2018-01'],
@@ -595,8 +566,11 @@ def period_range(
endpoints for a ``PeriodIndex`` with frequency matching that of the
``period_range`` constructor.
- >>> pd.period_range(start=pd.Period('2017Q1', freq='Q'),
- ... end=pd.Period('2017Q2', freq='Q'), freq='M')
+ >>> pd.period_range(
+ ... start=pd.Period("2017Q1", freq="Q"),
+ ... end=pd.Period("2017Q2", freq="Q"),
+ ... freq="M",
+ ... )
PeriodIndex(['2017-03', '2017-04', '2017-05', '2017-06'],
dtype='period[M]')
"""
diff --git a/pandas/core/indexes/range.py b/pandas/core/indexes/range.py
index 62afcf8badb50..0ba3c22093c69 100644
--- a/pandas/core/indexes/range.py
+++ b/pandas/core/indexes/range.py
@@ -22,15 +22,14 @@
index as libindex,
lib,
)
-from pandas._libs.algos import unique_deltas
from pandas._libs.lib import no_default
from pandas.compat.numpy import function as nv
from pandas.util._decorators import (
cache_readonly,
- deprecate_nonkeyword_arguments,
doc,
)
+from pandas.core.dtypes.base import ExtensionDtype
from pandas.core.dtypes.common import (
ensure_platform_int,
ensure_python_int,
@@ -44,6 +43,7 @@
from pandas.core import ops
import pandas.core.common as com
from pandas.core.construction import extract_array
+from pandas.core.indexers import check_array_indexer
import pandas.core.indexes.base as ibase
from pandas.core.indexes.base import (
Index,
@@ -55,6 +55,7 @@
from pandas._typing import (
Axis,
Dtype,
+ JoinHow,
NaPosition,
Self,
npt,
@@ -63,6 +64,12 @@
_dtype_int64 = np.dtype(np.int64)
+def min_fitting_element(start: int, step: int, lower_limit: int) -> int:
+ """Returns the smallest element greater than or equal to the limit"""
+ no_steps = -(-(lower_limit - start) // abs(step))
+ return start + abs(step) * no_steps
+
+
class RangeIndex(Index):
"""
Immutable Index implementing a monotonic integer range.
@@ -190,7 +197,7 @@ def from_range(cls, data: range, name=None, dtype: Dtype | None = None) -> Self:
if not isinstance(data, range):
raise TypeError(
f"{cls.__name__}(...) must be called with object coercible to a "
- f"range, {repr(data)} was passed"
+ f"range, {data!r} was passed"
)
cls._validate_dtype(dtype)
return cls._simple_new(data, name=name)
@@ -394,7 +401,7 @@ def __contains__(self, key: Any) -> bool:
hash(key)
try:
key = ensure_python_int(key)
- except TypeError:
+ except (TypeError, OverflowError):
return False
return key in self._range
@@ -470,28 +477,35 @@ def _shallow_copy(self, values, name: Hashable = no_default):
if values.dtype.kind == "f":
return Index(values, name=name, dtype=np.float64)
- # GH 46675 & 43885: If values is equally spaced, return a
- # more memory-compact RangeIndex instead of Index with 64-bit dtype
- unique_diffs = unique_deltas(values)
- if len(unique_diffs) == 1 and unique_diffs[0] != 0:
- diff = unique_diffs[0]
- new_range = range(values[0], values[-1] + diff, diff)
- return type(self)._simple_new(new_range, name=name)
- else:
- return self._constructor._simple_new(values, name=name)
+ if values.dtype.kind == "i" and values.ndim == 1:
+ # GH 46675 & 43885: If values is equally spaced, return a
+ # more memory-compact RangeIndex instead of Index with 64-bit dtype
+ if len(values) == 1:
+ start = values[0]
+ new_range = range(start, start + self.step, self.step)
+ return type(self)._simple_new(new_range, name=name)
+ maybe_range = ibase.maybe_sequence_to_range(values)
+ if isinstance(maybe_range, range):
+ return type(self)._simple_new(maybe_range, name=name)
+ return self._constructor._simple_new(values, name=name)
def _view(self) -> Self:
result = type(self)._simple_new(self._range, name=self._name)
result._cache = self._cache
return result
+ def _wrap_reindex_result(self, target, indexer, preserve_names: bool):
+ if not isinstance(target, type(self)) and target.dtype.kind == "i":
+ target = self._shallow_copy(target._values, name=target.name)
+ return super()._wrap_reindex_result(target, indexer, preserve_names)
+
@doc(Index.copy)
def copy(self, name: Hashable | None = None, deep: bool = False) -> Self:
name = self._validate_names(name=name, deep=deep)[0]
new_index = self._rename(name=name)
return new_index
- def _minmax(self, meth: str):
+ def _minmax(self, meth: Literal["min", "max"]) -> int | float:
no_steps = len(self) - 1
if no_steps == -1:
return np.nan
@@ -500,18 +514,51 @@ def _minmax(self, meth: str):
return self.start + self.step * no_steps
- def min(self, axis=None, skipna: bool = True, *args, **kwargs) -> int:
+ def min(self, axis=None, skipna: bool = True, *args, **kwargs) -> int | float:
"""The minimum value of the RangeIndex"""
nv.validate_minmax_axis(axis)
nv.validate_min(args, kwargs)
return self._minmax("min")
- def max(self, axis=None, skipna: bool = True, *args, **kwargs) -> int:
+ def max(self, axis=None, skipna: bool = True, *args, **kwargs) -> int | float:
"""The maximum value of the RangeIndex"""
nv.validate_minmax_axis(axis)
nv.validate_max(args, kwargs)
return self._minmax("max")
+ def _argminmax(
+ self,
+ meth: Literal["min", "max"],
+ axis=None,
+ skipna: bool = True,
+ ) -> int:
+ nv.validate_minmax_axis(axis)
+ if len(self) == 0:
+ return getattr(super(), f"arg{meth}")(
+ axis=axis,
+ skipna=skipna,
+ )
+ elif meth == "min":
+ if self.step > 0:
+ return 0
+ else:
+ return len(self) - 1
+ elif meth == "max":
+ if self.step > 0:
+ return len(self) - 1
+ else:
+ return 0
+ else:
+ raise ValueError(f"{meth=} must be max or min")
+
+ def argmin(self, axis=None, skipna: bool = True, *args, **kwargs) -> int:
+ nv.validate_argmin(args, kwargs)
+ return self._argminmax("min", axis=axis, skipna=skipna)
+
+ def argmax(self, axis=None, skipna: bool = True, *args, **kwargs) -> int:
+ nv.validate_argmax(args, kwargs)
+ return self._argminmax("max", axis=axis, skipna=skipna)
+
def argsort(self, *args, **kwargs) -> npt.NDArray[np.intp]:
"""
Returns the indices that would sort the index and its
@@ -529,25 +576,30 @@ def argsort(self, *args, **kwargs) -> npt.NDArray[np.intp]:
kwargs.pop("kind", None) # e.g. "mergesort" is irrelevant
nv.validate_argsort(args, kwargs)
+ start, stop, step = None, None, None
if self._range.step > 0:
- result = np.arange(len(self), dtype=np.intp)
+ if ascending:
+ start = len(self)
+ else:
+ start, stop, step = len(self) - 1, -1, -1
+ elif ascending:
+ start, stop, step = len(self) - 1, -1, -1
else:
- result = np.arange(len(self) - 1, -1, -1, dtype=np.intp)
+ start = len(self)
- if not ascending:
- result = result[::-1]
- return result
+ return np.arange(start, stop, step, dtype=np.intp)
def factorize(
self,
sort: bool = False,
use_na_sentinel: bool = True,
) -> tuple[npt.NDArray[np.intp], RangeIndex]:
- codes = np.arange(len(self), dtype=np.intp)
- uniques = self
if sort and self.step < 0:
- codes = codes[::-1]
- uniques = uniques[::-1]
+ codes = np.arange(len(self) - 1, -1, -1, dtype=np.intp)
+ uniques = self[::-1]
+ else:
+ codes = np.arange(len(self), dtype=np.intp)
+ uniques = self
return codes, uniques
def equals(self, other: object) -> bool:
@@ -567,8 +619,7 @@ def sort_values(
ascending: bool = ...,
na_position: NaPosition = ...,
key: Callable | None = ...,
- ) -> Self:
- ...
+ ) -> Self: ...
@overload
def sort_values(
@@ -578,8 +629,7 @@ def sort_values(
ascending: bool = ...,
na_position: NaPosition = ...,
key: Callable | None = ...,
- ) -> tuple[Self, np.ndarray | RangeIndex]:
- ...
+ ) -> tuple[Self, np.ndarray | RangeIndex]: ...
@overload
def sort_values(
@@ -589,14 +639,11 @@ def sort_values(
ascending: bool = ...,
na_position: NaPosition = ...,
key: Callable | None = ...,
- ) -> Self | tuple[Self, np.ndarray | RangeIndex]:
- ...
+ ) -> Self | tuple[Self, np.ndarray | RangeIndex]: ...
- @deprecate_nonkeyword_arguments(
- version="3.0", allowed_args=["self"], name="sort_values"
- )
def sort_values(
self,
+ *,
return_indexer: bool = False,
ascending: bool = True,
na_position: NaPosition = "last",
@@ -663,26 +710,15 @@ def _intersection(self, other: Index, sort: bool = False):
# intersection disregarding the lower bounds
tmp_start = first.start + (second.start - first.start) * first.step // gcd * s
new_step = first.step * second.step // gcd
- new_range = range(tmp_start, int_high, new_step)
- new_index = self._simple_new(new_range)
# adjust index to limiting interval
- new_start = new_index._min_fitting_element(int_low)
- new_range = range(new_start, new_index.stop, new_index.step)
- new_index = self._simple_new(new_range)
-
- if (self.step < 0 and other.step < 0) is not (new_index.step < 0):
- new_index = new_index[::-1]
+ new_start = min_fitting_element(tmp_start, new_step, int_low)
+ new_range = range(new_start, int_high, new_step)
- if sort is None:
- new_index = new_index.sort_values()
+ if (self.step < 0 and other.step < 0) is not (new_range.step < 0):
+ new_range = new_range[::-1]
- return new_index
-
- def _min_fitting_element(self, lower_limit: int) -> int:
- """Returns the smallest element greater than or equal to the limit"""
- no_steps = -(-(lower_limit - self.start) // abs(self.step))
- return self.start + abs(self.step) * no_steps
+ return self._simple_new(new_range)
def _extended_gcd(self, a: int, b: int) -> tuple[int, int, int]:
"""
@@ -868,15 +904,15 @@ def _difference(self, other, sort=None):
# e.g. range(10) and range(0, 10, 3)
return super()._difference(other, sort=sort)
- new_index = type(self)._simple_new(new_rng, name=res_name)
if first is not self._range:
- new_index = new_index[::-1]
+ new_rng = new_rng[::-1]
+ new_index = type(self)._simple_new(new_rng, name=res_name)
return new_index
def symmetric_difference(
self, other, result_name: Hashable | None = None, sort=None
- ):
+ ) -> Index:
if not isinstance(other, RangeIndex) or sort is not None:
return super().symmetric_difference(other, result_name, sort)
@@ -888,6 +924,48 @@ def symmetric_difference(
result = result.rename(result_name)
return result
+ def _join_empty(
+ self, other: Index, how: JoinHow, sort: bool
+ ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
+ if not isinstance(other, RangeIndex) and other.dtype.kind == "i":
+ other = self._shallow_copy(other._values, name=other.name)
+ return super()._join_empty(other, how=how, sort=sort)
+
+ def _join_monotonic(
+ self, other: Index, how: JoinHow = "left"
+ ) -> tuple[Index, npt.NDArray[np.intp] | None, npt.NDArray[np.intp] | None]:
+ # This currently only gets called for the monotonic increasing case
+ if not isinstance(other, type(self)):
+ maybe_ri = self._shallow_copy(other._values, name=other.name)
+ if not isinstance(maybe_ri, type(self)):
+ return super()._join_monotonic(other, how=how)
+ other = maybe_ri
+
+ if self.equals(other):
+ ret_index = other if how == "right" else self
+ return ret_index, None, None
+
+ if how == "left":
+ join_index = self
+ lidx = None
+ ridx = other.get_indexer(join_index)
+ elif how == "right":
+ join_index = other
+ lidx = self.get_indexer(join_index)
+ ridx = None
+ elif how == "inner":
+ join_index = self.intersection(other)
+ lidx = self.get_indexer(join_index)
+ ridx = other.get_indexer(join_index)
+ elif how == "outer":
+ join_index = self.union(other)
+ lidx = self.get_indexer(join_index)
+ ridx = other.get_indexer(join_index)
+
+ lidx = None if lidx is None else ensure_platform_int(lidx)
+ ridx = None if ridx is None else ensure_platform_int(ridx)
+ return join_index, lidx, ridx
+
# --------------------------------------------------------------------
# error: Return type "Index" of "delete" incompatible with return type
@@ -915,23 +993,27 @@ def delete(self, loc) -> Index: # type: ignore[override]
return super().delete(loc)
def insert(self, loc: int, item) -> Index:
- if len(self) and (is_integer(item) or is_float(item)):
+ if is_integer(item) or is_float(item):
# We can retain RangeIndex is inserting at the beginning or end,
# or right in the middle.
- rng = self._range
- if loc == 0 and item == self[0] - self.step:
- new_rng = range(rng.start - rng.step, rng.stop, rng.step)
- return type(self)._simple_new(new_rng, name=self._name)
-
- elif loc == len(self) and item == self[-1] + self.step:
- new_rng = range(rng.start, rng.stop + rng.step, rng.step)
- return type(self)._simple_new(new_rng, name=self._name)
-
- elif len(self) == 2 and item == self[0] + self.step / 2:
- # e.g. inserting 1 into [0, 2]
- step = int(self.step / 2)
- new_rng = range(self.start, self.stop, step)
+ if len(self) == 0 and loc == 0 and is_integer(item):
+ new_rng = range(item, item + self.step, self.step)
return type(self)._simple_new(new_rng, name=self._name)
+ elif len(self):
+ rng = self._range
+ if loc == 0 and item == self[0] - self.step:
+ new_rng = range(rng.start - rng.step, rng.stop, rng.step)
+ return type(self)._simple_new(new_rng, name=self._name)
+
+ elif loc == len(self) and item == self[-1] + self.step:
+ new_rng = range(rng.start, rng.stop + rng.step, rng.step)
+ return type(self)._simple_new(new_rng, name=self._name)
+
+ elif len(self) == 2 and item == self[0] + self.step / 2:
+ # e.g. inserting 1 into [0, 2]
+ step = int(self.step / 2)
+ new_rng = range(self.start, self.stop, step)
+ return type(self)._simple_new(new_rng, name=self._name)
return super().insert(loc, item)
@@ -945,7 +1027,10 @@ def _concat(self, indexes: list[Index], name: Hashable) -> Index:
indexes = [RangeIndex(3), RangeIndex(4, 6)] -> Index([0,1,2,4,5], dtype='int64')
"""
if not all(isinstance(x, RangeIndex) for x in indexes):
- return super()._concat(indexes, name)
+ result = super()._concat(indexes, name)
+ if result.dtype.kind == "i":
+ return self._shallow_copy(result._values)
+ return result
elif len(indexes) == 1:
return indexes[0]
@@ -955,7 +1040,17 @@ def _concat(self, indexes: list[Index], name: Hashable) -> Index:
start = step = next_ = None
# Filter the empty indexes
- non_empty_indexes = [obj for obj in rng_indexes if len(obj)]
+ non_empty_indexes = []
+ all_same_index = True
+ prev: RangeIndex | None = None
+ for obj in rng_indexes:
+ if len(obj):
+ non_empty_indexes.append(obj)
+ if all_same_index:
+ if prev is not None:
+ all_same_index = prev.equals(obj)
+ else:
+ prev = obj
for obj in non_empty_indexes:
rng = obj._range
@@ -968,7 +1063,12 @@ def _concat(self, indexes: list[Index], name: Hashable) -> Index:
elif step is None:
# First non-empty index had only one element
if rng.start == start:
- values = np.concatenate([x._values for x in rng_indexes])
+ if all_same_index:
+ values = np.tile(
+ non_empty_indexes[0]._values, len(non_empty_indexes)
+ )
+ else:
+ values = np.concatenate([x._values for x in rng_indexes])
result = self._constructor(values)
return result.rename(name)
@@ -978,9 +1078,13 @@ def _concat(self, indexes: list[Index], name: Hashable) -> Index:
next_ is not None and rng.start != next_
)
if non_consecutive:
- result = self._constructor(
- np.concatenate([x._values for x in rng_indexes])
- )
+ if all_same_index:
+ values = np.tile(
+ non_empty_indexes[0]._values, len(non_empty_indexes)
+ )
+ else:
+ values = np.concatenate([x._values for x in rng_indexes])
+ result = self._constructor(values)
return result.rename(name)
if step is not None:
@@ -990,11 +1094,13 @@ def _concat(self, indexes: list[Index], name: Hashable) -> Index:
# Get the stop value from "next" or alternatively
# from the last non-empty index
stop = non_empty_indexes[-1].stop if next_ is None else next_
- return RangeIndex(start, stop, step).rename(name)
+ if len(non_empty_indexes) == 1:
+ step = non_empty_indexes[0].step
+ return RangeIndex(start, stop, step, name=name)
# Here all "indexes" had 0 length, i.e. were empty.
# In this case return an empty range index.
- return RangeIndex(0, 0).rename(name)
+ return RangeIndex(_empty_range, name=name)
def __len__(self) -> int:
"""
@@ -1010,6 +1116,8 @@ def __getitem__(self, key):
"""
Conserve RangeIndex type for scalar and slice keys.
"""
+ if key is Ellipsis:
+ key = slice(None)
if isinstance(key, slice):
return self._getitem_slice(key)
elif is_integer(key):
@@ -1027,7 +1135,17 @@ def __getitem__(self, key):
"and integer or boolean "
"arrays are valid indices"
)
- return super().__getitem__(key)
+ elif com.is_bool_indexer(key):
+ if isinstance(getattr(key, "dtype", None), ExtensionDtype):
+ key = key.to_numpy(dtype=bool, na_value=False)
+ else:
+ key = np.asarray(key, dtype=bool)
+ check_array_indexer(self._range, key) # type: ignore[arg-type]
+ key = np.flatnonzero(key)
+ try:
+ return self.take(key)
+ except (TypeError, ValueError):
+ return super().__getitem__(key)
def _getitem_slice(self, slobj: slice) -> Self:
"""
@@ -1063,6 +1181,42 @@ def any(self, *args, **kwargs) -> bool:
# --------------------------------------------------------------------
+ # error: Return type "RangeIndex | Index" of "round" incompatible with
+ # return type "RangeIndex" in supertype "Index"
+ def round(self, decimals: int = 0) -> Self | Index: # type: ignore[override]
+ """
+ Round each value in the Index to the given number of decimals.
+
+ Parameters
+ ----------
+ decimals : int, optional
+ Number of decimal places to round to. If decimals is negative,
+ it specifies the number of positions to the left of the decimal point
+ e.g. ``round(11.0, -1) == 10.0``.
+
+ Returns
+ -------
+ Index or RangeIndex
+ A new Index with the rounded values.
+
+ Examples
+ --------
+ >>> import pandas as pd
+ >>> idx = pd.RangeIndex(10, 30, 10)
+ >>> idx.round(decimals=-1)
+ RangeIndex(start=10, stop=30, step=10)
+ >>> idx = pd.RangeIndex(10, 15, 1)
+ >>> idx.round(decimals=-1)
+ Index([10, 10, 10, 10, 10], dtype='int64')
+ """
+ if decimals >= 0:
+ return self.copy()
+ elif self.start % 10**-decimals == 0 and self.step % 10**-decimals == 0:
+ # e.g. RangeIndex(10, 30, 10).round(-1) doesn't need rounding
+ return self.copy()
+ else:
+ return super().round(decimals=decimals)
+
def _cmp_method(self, other, op):
if isinstance(other, RangeIndex) and self._range == other._range:
# Both are immutable so if ._range attr. are equal, shortcut is possible
@@ -1143,6 +1297,27 @@ def _arith_method(self, other, op):
# test_arithmetic_explicit_conversions
return super()._arith_method(other, op)
+ def __abs__(self) -> Self | Index:
+ if len(self) == 0 or self.min() >= 0:
+ return self.copy()
+ elif self.max() <= 0:
+ return -self
+ else:
+ return super().__abs__()
+
+ def __neg__(self) -> Self:
+ rng = range(-self.start, -self.stop, -self.step)
+ return self._simple_new(rng, name=self.name)
+
+ def __pos__(self) -> Self:
+ return self.copy()
+
+ def __invert__(self) -> Self:
+ if len(self) == 0:
+ return self.copy()
+ rng = range(~self.start, ~self.stop, -self.step)
+ return self._simple_new(rng, name=self.name)
+
# error: Return type "Index" of "take" incompatible with return type
# "RangeIndex" in supertype "Index"
def take( # type: ignore[override]
@@ -1152,7 +1327,7 @@ def take( # type: ignore[override]
allow_fill: bool = True,
fill_value=None,
**kwargs,
- ) -> Index:
+ ) -> Self | Index:
if kwargs:
nv.validate_take((), kwargs)
if is_scalar(indices):
@@ -1163,7 +1338,7 @@ def take( # type: ignore[override]
self._maybe_disallow_fill(allow_fill, fill_value, indices)
if len(indices) == 0:
- taken = np.array([], dtype=self.dtype)
+ return type(self)(_empty_range, name=self.name)
else:
ind_max = indices.max()
if ind_max >= len(self):
@@ -1183,5 +1358,4 @@ def take( # type: ignore[override]
if self.start != 0:
taken += self.start
- # _constructor so RangeIndex-> Index with an int64 dtype
- return self._constructor._simple_new(taken, name=self.name)
+ return self._shallow_copy(taken, name=self.name)
diff --git a/pandas/core/indexes/timedeltas.py b/pandas/core/indexes/timedeltas.py
index 08a265ba47648..8af5a56f43c57 100644
--- a/pandas/core/indexes/timedeltas.py
+++ b/pandas/core/indexes/timedeltas.py
@@ -1,8 +1,8 @@
-""" implement the TimedeltaIndex """
+"""implement the TimedeltaIndex"""
+
from __future__ import annotations
from typing import TYPE_CHECKING
-import warnings
from pandas._libs import (
index as libindex,
@@ -13,8 +13,6 @@
Timedelta,
to_offset,
)
-from pandas._libs.tslibs.timedeltas import disallow_ambiguous_unit
-from pandas.util._exceptions import find_stack_level
from pandas.core.dtypes.common import (
is_scalar,
@@ -32,6 +30,7 @@
from pandas.core.indexes.extension import inherit_names
if TYPE_CHECKING:
+ from pandas._libs import NaTType
from pandas._typing import DtypeObj
@@ -61,12 +60,6 @@ class TimedeltaIndex(DatetimeTimedeltaMixin):
----------
data : array-like (1-dimensional), optional
Optional timedelta-like data to construct index with.
- unit : {'D', 'h', 'm', 's', 'ms', 'us', 'ns'}, optional
- The unit of ``data``.
-
- .. deprecated:: 2.2.0
- Use ``pd.to_timedelta`` instead.
-
freq : str or pandas offset object, optional
One of pandas date offset strings or corresponding objects. The string
``'infer'`` can be passed in order to set the frequency of the index as
@@ -108,18 +101,18 @@ class TimedeltaIndex(DatetimeTimedeltaMixin):
Notes
-----
- To learn more about the frequency strings, please see `this link
- `__.
+ To learn more about the frequency strings, please see
+ :ref:`this link`.
Examples
--------
- >>> pd.TimedeltaIndex(['0 days', '1 days', '2 days', '3 days', '4 days'])
+ >>> pd.TimedeltaIndex(["0 days", "1 days", "2 days", "3 days", "4 days"])
TimedeltaIndex(['0 days', '1 days', '2 days', '3 days', '4 days'],
dtype='timedelta64[ns]', freq=None)
We can also let pandas infer the frequency when possible.
- >>> pd.TimedeltaIndex(np.arange(5) * 24 * 3600 * 1e9, freq='infer')
+ >>> pd.TimedeltaIndex(np.arange(5) * 24 * 3600 * 1e9, freq="infer")
TimedeltaIndex(['0 days', '1 days', '2 days', '3 days', '4 days'],
dtype='timedelta64[ns]', freq='D')
"""
@@ -149,40 +142,16 @@ def _resolution_obj(self) -> Resolution | None: # type: ignore[override]
def __new__(
cls,
data=None,
- unit=lib.no_default,
freq=lib.no_default,
- closed=lib.no_default,
dtype=None,
copy: bool = False,
name=None,
):
- if closed is not lib.no_default:
- # GH#52628
- warnings.warn(
- f"The 'closed' keyword in {cls.__name__} construction is "
- "deprecated and will be removed in a future version.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
-
- if unit is not lib.no_default:
- # GH#55499
- warnings.warn(
- f"The 'unit' keyword in {cls.__name__} construction is "
- "deprecated and will be removed in a future version. "
- "Use pd.to_timedelta instead.",
- FutureWarning,
- stacklevel=find_stack_level(),
- )
- else:
- unit = None
-
name = maybe_extract_name(name, data, cls)
if is_scalar(data):
cls._raise_scalar_data_error(data)
- disallow_ambiguous_unit(unit)
if dtype is not None:
dtype = pandas_dtype(dtype)
@@ -209,7 +178,7 @@ def __new__(
# - Cases checked above all return/raise before reaching here - #
tdarr = TimedeltaArray._from_sequence_not_strict(
- data, freq=freq, unit=unit, dtype=dtype, copy=copy
+ data, freq=freq, unit=None, dtype=dtype, copy=copy
)
refs = None
if not copy and isinstance(data, (ABCSeries, Index)):
@@ -245,7 +214,10 @@ def get_loc(self, key):
return Index.get_loc(self, key)
- def _parse_with_reso(self, label: str):
+ # error: Return type "tuple[Timedelta | NaTType, None]" of "_parse_with_reso"
+ # incompatible with return type "tuple[datetime, Resolution]" in supertype
+ # "DatetimeIndexOpsMixin"
+ def _parse_with_reso(self, label: str) -> tuple[Timedelta | NaTType, None]: # type: ignore[override]
# the "with_reso" is a no-op for TimedeltaIndex
parsed = Timedelta(label)
return parsed, None
@@ -299,6 +271,7 @@ def timedelta_range(
Returns
-------
TimedeltaIndex
+ Fixed frequency, with day as the default.
Notes
-----
@@ -307,19 +280,19 @@ def timedelta_range(
``TimedeltaIndex`` will have ``periods`` linearly spaced elements between
``start`` and ``end`` (closed on both sides).
- To learn more about the frequency strings, please see `this link
- `__.
+ To learn more about the frequency strings, please see
+ :ref:`this link`.
Examples
--------
- >>> pd.timedelta_range(start='1 day', periods=4)
+ >>> pd.timedelta_range(start="1 day", periods=4)
TimedeltaIndex(['1 days', '2 days', '3 days', '4 days'],
dtype='timedelta64[ns]', freq='D')
The ``closed`` parameter specifies which endpoint is included. The default
behavior is to include both endpoints.
- >>> pd.timedelta_range(start='1 day', periods=4, closed='right')
+ >>> pd.timedelta_range(start="1 day", periods=4, closed="right")
TimedeltaIndex(['2 days', '3 days', '4 days'],
dtype='timedelta64[ns]', freq='D')
@@ -327,7 +300,7 @@ def timedelta_range(
Only fixed frequencies can be passed, non-fixed frequencies such as
'M' (month end) will raise.
- >>> pd.timedelta_range(start='1 day', end='2 days', freq='6h')
+ >>> pd.timedelta_range(start="1 day", end="2 days", freq="6h")
TimedeltaIndex(['1 days 00:00:00', '1 days 06:00:00', '1 days 12:00:00',
'1 days 18:00:00', '2 days 00:00:00'],
dtype='timedelta64[ns]', freq='6h')
@@ -335,7 +308,7 @@ def timedelta_range(
Specify ``start``, ``end``, and ``periods``; the frequency is generated
automatically (linearly spaced).
- >>> pd.timedelta_range(start='1 day', end='5 days', periods=4)
+ >>> pd.timedelta_range(start="1 day", end="5 days", periods=4)
TimedeltaIndex(['1 days 00:00:00', '2 days 08:00:00', '3 days 16:00:00',
'5 days 00:00:00'],
dtype='timedelta64[ns]', freq=None)
diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py
index a7dd3b486ab11..982e305b7e471 100644
--- a/pandas/core/indexing.py
+++ b/pandas/core/indexing.py
@@ -13,11 +13,6 @@
import numpy as np
-from pandas._config import (
- using_copy_on_write,
- warn_copy_on_write,
-)
-
from pandas._libs.indexing import NDFrameIndexerBase
from pandas._libs.lib import item_from_zerodim
from pandas.compat import PYPY
@@ -27,10 +22,8 @@
IndexingError,
InvalidIndexError,
LossySetitemError,
- _chained_assignment_msg,
- _chained_assignment_warning_msg,
- _check_cacher,
)
+from pandas.errors.cow import _chained_assignment_msg
from pandas.util._decorators import doc
from pandas.util._exceptions import find_stack_level
@@ -57,6 +50,7 @@
ABCSeries,
)
from pandas.core.dtypes.missing import (
+ construct_1d_array_from_inferred_fill_value,
infer_fill_value,
is_valid_na_for_dtype,
isna,
@@ -68,7 +62,6 @@
from pandas.core.construction import (
array as pd_array,
extract_array,
- sanitize_array,
)
from pandas.core.indexers import (
check_array_indexer,
@@ -121,14 +114,17 @@ class _IndexSlice:
Examples
--------
- >>> midx = pd.MultiIndex.from_product([['A0','A1'], ['B0','B1','B2','B3']])
- >>> columns = ['foo', 'bar']
- >>> dfmi = pd.DataFrame(np.arange(16).reshape((len(midx), len(columns))),
- ... index=midx, columns=columns)
+ >>> midx = pd.MultiIndex.from_product([["A0", "A1"], ["B0", "B1", "B2", "B3"]])
+ >>> columns = ["foo", "bar"]
+ >>> dfmi = pd.DataFrame(
+ ... np.arange(16).reshape((len(midx), len(columns))),
+ ... index=midx,
+ ... columns=columns,
+ ... )
Using the default slice command:
- >>> dfmi.loc[(slice(None), slice('B0', 'B1')), :]
+ >>> dfmi.loc[(slice(None), slice("B0", "B1")), :]
foo bar
A0 B0 0 1
B1 2 3
@@ -138,7 +134,7 @@ class _IndexSlice:
Using the IndexSlice class for a more intuitive command:
>>> idx = pd.IndexSlice
- >>> dfmi.loc[idx[:, 'B0':'B1'], :]
+ >>> dfmi.loc[idx[:, "B0":"B1"], :]
foo bar
A0 B0 0 1
B1 2 3
@@ -200,9 +196,11 @@ def iloc(self) -> _iLocIndexer:
Examples
--------
- >>> mydict = [{'a': 1, 'b': 2, 'c': 3, 'd': 4},
- ... {'a': 100, 'b': 200, 'c': 300, 'd': 400},
- ... {'a': 1000, 'b': 2000, 'c': 3000, 'd': 4000}]
+ >>> mydict = [
+ ... {"a": 1, "b": 2, "c": 3, "d": 4},
+ ... {"a": 100, "b": 200, "c": 300, "d": 400},
+ ... {"a": 1000, "b": 2000, "c": 3000, "d": 4000},
+ ... ]
>>> df = pd.DataFrame(mydict)
>>> df
a b c d
@@ -229,7 +227,7 @@ def iloc(self) -> _iLocIndexer:
a b c d
0 1 2 3 4
>>> type(df.iloc[[0]])
-
+