change pandas engine to be numpy>2 compat (#1690)

* change pandas engine to be numpy>2 compat Signed-off-by: Niels Bantilan <[email protected]> * remove numpy from mypy pre-commit type stubs Signed-off-by: Niels Bantilan <[email protected]> * install numpy<2.0.0 in readthedocs build Signed-off-by: Niels Bantilan <[email protected]> * remove check docs ci test Signed-off-by: Niels Bantilan <[email protected]> * update Signed-off-by: Niels Bantilan <[email protected]> * debug Signed-off-by: Niels Bantilan <[email protected]> * update requirements Signed-off-by: Niels Bantilan <[email protected]> * update requirements, only use numpy2 for latest pandas Signed-off-by: Niels Bantilan <[email protected]> * move pyspark config Signed-off-by: Niels Bantilan <[email protected]> * remove typing from root __init__ Signed-off-by: Niels Bantilan <[email protected]> * [wip] try patching numpy2 with NaN Signed-off-by: cosmicBboy <[email protected]> * [wip] remove pyspark import from typing.pandas Signed-off-by: cosmicBboy <[email protected]> * isolate dask, modin, pyspark, etc imports, update ci Signed-off-by: cosmicBboy <[email protected]> * update numpy patch Signed-off-by: cosmicBboy <[email protected]> * fix ci numpy pin Signed-off-by: cosmicBboy <[email protected]> * update ci Signed-off-by: cosmicBboy <[email protected]> * update Signed-off-by: cosmicBboy <[email protected]> * update Signed-off-by: cosmicBboy <[email protected]> * debug Signed-off-by: cosmicBboy <[email protected]> * debug Signed-off-by: cosmicBboy <[email protected]> * debug Signed-off-by: cosmicBboy <[email protected]> * update ci test for mypy Signed-off-by: cosmicBboy <[email protected]> * add numpy patch to root pandera Signed-off-by: cosmicBboy <[email protected]> * debug Signed-off-by: cosmicBboy <[email protected]> * debug Signed-off-by: cosmicBboy <[email protected]> * debug Signed-off-by: cosmicBboy <[email protected]> * skip modin test for decimals, remove pandas2.0.3 Signed-off-by: cosmicBboy <[email protected]> --------- Signed-off-by: Niels Bantilan <[email protected]> Signed-off-by: cosmicBboy <[email protected]>
unionai-oss · Jun 20, 2024 · 47c236e · 47c236e
1 parent efbaa91
commit 47c236e
Show file tree

Hide file tree

Showing 44 changed files with 3,754 additions and 6,183 deletions.
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
@@ -87,7 +87,7 @@ jobs:
 
   tests:
     name: >
-      CI Tests (${{ matrix.python-version }}, ${{ matrix.os }}, pandas-${{ matrix.pandas-version }}, pydantic-${{ matrix.pydantic-version }})
+      CI Core Tests (${{ matrix.python-version }}, ${{ matrix.os }}, pandas-${{ matrix.pandas-version }}, pydantic-${{ matrix.pydantic-version }})
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -106,7 +106,7 @@ jobs:
         - macos-13
         # - macos-latest  # see: https://github.com/actions/setup-python/issues/696
         python-version: ["3.8", "3.9", "3.10", "3.11"]
-        pandas-version: ["1.5.3", "2.0.3", "2.2.0"]
+        pandas-version: ["1.5.3", "2.2.2"]
         pydantic-version: ["1.10.11", "2.3.0"]
         include:
         - os: ubuntu-latest
@@ -119,7 +119,7 @@ jobs:
           pip-cache: ~/AppData/Local/pip/Cache
         exclude:
         - python-version: "3.8"
-          pandas-version: "2.2.0"
+          pandas-version: "2.2.2"
         - python-version: "3.11"
           pandas-version: "1.5.3"
 
@@ -135,14 +135,14 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ${{ matrix.pip-cache }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.in') }}
           restore-keys: |
             ${{ runner.os }}-pip-
 
       - name: Install deps
         run:
           pip install uv
-          uv pip install -r ci/requirements-py${{ matrix.python-version }}-pandas${{ matrix.pandas-version }}-pydantic${{ matrix.pydantic-version }}.txt
+          uv pip install -r requirements.in
 
       - run: |
           pip list
@@ -151,51 +151,102 @@ jobs:
       - name: Unit Tests - Core
         run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='core', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
-      - name: Unit Tests - Hypotheses
-        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='hypotheses', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
 
-      - name: Unit Tests - IO
-        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='io', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
+      # - name: Check Docstrings
+      #   if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pandas-version == '2.2.0' }}
+      #   run: nox ${{ env.NOX_FLAGS }} --session doctests
 
-      - name: Unit Tests - Mypy
-        # mypy tests hang on windows
-        if: ${{ matrix.os != 'windows-latest' }}
-        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='mypy', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
+      # - name: Check Docs
+      #   if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pydantic-version == '2.2.0' }}
+      #   run: nox ${{ env.NOX_FLAGS }} --session docs
 
-      - name: Unit Tests - Strategies
-        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='strategies', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
+  extras-tests:
+    name: >
+      CI Extras Tests: ${{ matrix.extra }} (${{ matrix.python-version }}, ${{ matrix.os }}, pandas-${{ matrix.pandas-version }}, pydantic-${{ matrix.pydantic-version }})
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: pwsh
+    env:
+      PYTHONUTF8: 1
+      NOX_FLAGS: -v --no-install --no-venv --non-interactive
+      PYTEST_FLAGS: --cov=pandera --cov-report=term-missing --cov-report=xml --cov-append
+      HYPOTHESIS_FLAGS: -n=auto -q --hypothesis-profile=ci
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+        - ubuntu-latest
+        - windows-latest
+        - macos-13
+        # - macos-latest  # see: https://github.com/actions/setup-python/issues/696
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        pandas-version: ["2.2.2"]
+        pydantic-version: ["2.3.0"]
+        extra:
+        - hypotheses
+        - io
+        - mypy
+        - strategies
+        - fastapi
+        - geopandas
+        - dask
+        - pyspark
+        - modin-dask
+        - modin-ray
+        include:
+        - os: ubuntu-latest
+          pip-cache: ~/.cache/pip
+        # - os: macos-latest
+        #   pip-cache: ~/Library/Caches/pip
+        - os: macos-13
+          pip-cache: ~/Library/Caches/pip
+        - os: windows-latest
+          pip-cache: ~/AppData/Local/pip/Cache
+        exclude:
+        - python-version: "3.8"
+          pandas-version: "2.2.0"
+        - python-version: "3.11"
+          pandas-version: "1.5.3"
+          # mypy tests hang on windows
+        - extra: mypy
+          os: windows-latest
+        - extra: modin-ray
+          os: windows-latest
 
-      - name: Unit Tests - FastAPI
-        # there's an issue with the fastapi tests in CI that's not reproducible locally
-        # when pydantic > v2
-        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='fastapi', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
+    steps:
+      - uses: actions/checkout@v4
 
-      - name: Unit Tests - GeoPandas
-        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='geopandas', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
 
-      - name: Unit Tests - Dask
-        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='dask', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ${{ matrix.pip-cache }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
 
-      - name: Unit Tests - Pyspark
-        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='pyspark', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
+      - name: Install deps
+        shell: bash
+        # install numpy<2 for all extras. It seems to effects most packages in
+        # the ecosystem.
+        run: |
+          pip install uv
+          uv pip install --system -r requirements.in
+          uv pip install --system --force-reinstall "numpy<2.0.0";
 
-      - name: Unit Tests - Modin-Dask
-        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='modin-dask', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
+      - run: |
+          pip list
+          printenv | sort
 
-      - name: Unit Tests - Modin-Ray
-        # ray CI issues with the following:
-        # - windows, python 3.10
-        # Tracking issue: https://github.com/modin-project/modin/issues/5466
-        if: ${{ matrix.os != 'windows-latest' }}
-        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='modin-ray', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
+      - name: Unit Tests - ${{ matrix.extra }}
+        run: nox ${{ env.NOX_FLAGS }} --session "tests(extra='${{ matrix.extra }}', pydantic='${{ matrix.pydantic-version }}', python='${{ matrix.python-version }}', pandas='${{ matrix.pandas-version }}')"
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
-
-      - name: Check Docstrings
-        if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pandas-version == '2.2.0' }}
-        run: nox ${{ env.NOX_FLAGS }} --session doctests
-
-      - name: Check Docs
-        if: ${{ matrix.os != 'windows-latest' && matrix.python-version == '3.11' && matrix.pydantic-version == '2.2.0' }}
-        run: nox ${{ env.NOX_FLAGS }} --session docs
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -49,7 +49,6 @@ repos:
     hooks:
       - id: mypy
         additional_dependencies:
-          - numpy
           - types-click
           - types-pkg_resources
           - types-pytz

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -9,6 +9,11 @@ build:
   os: ubuntu-20.04
   tools:
     python: "mambaforge-4.10"
+  jobs:
+    post_install:
+      # Install numpy<v2 to avoid breaking docs. This is because the env has
+      # pyspark installed, which requires numpy<2.
+      - python -m pip install "numpy<2"
 
 # Build documentation in the docs/ directory with Sphinx
 conda: