feat: Python 3.13 is officially supported (#2518)

* test: Start testing with Python 3.13 * Let tests run * Bump duckdb to get Python 3.13 wheels * Update xfail markers * Ignore SQLite `ResourceWarnings` * Update templates * Update more xfail markers * There are some XPASS * Update max Py version in `--about` * Use latest Python where possible * Allow some sessions to run on any Python * Fix external tests * Restore 3.12 as main Python version * Let mypy and deps run * Use default latest Python for extra sessions * Revert "Use default latest Python for extra sessions" This reverts commit a03bae8.
meltano · Oct 18, 2024 · 4454aee · 4454aee
1 parent dac0305
commit 4454aee
Show file tree

Hide file tree

Showing 23 changed files with 89 additions and 36 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -32,6 +32,7 @@ body:
       description: Version of Python you are using
       options:
         - "NA"
+        - "3.13"
         - "3.12"
         - "3.11"
         - "3.10"

diff --git a/.github/workflows/api-changes.yml b/.github/workflows/api-changes.yml
@@ -30,7 +30,7 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v5
       with:
-        python-version: 3.12
+        python-version: 3.x
 
     - name: Install tools
       env:

diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -30,7 +30,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: 3.12
+        python-version: 3.x
         architecture: x64
 
     - name: Install poetry

diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml
@@ -24,14 +24,8 @@ env:
 
 jobs:
   lint:
-    name: Cookiecutter E2E Python ${{ matrix.python-version }} / ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - { python-version: "3.12",  os: "ubuntu-latest" }
-
+    name: Cookiecutter E2E Python
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
     - name: Upgrade pip
@@ -50,8 +44,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
-        architecture: x64
+        python-version: 3.x
         cache: 'pip'
         cache-dependency-path: 'poetry.lock'
 
@@ -69,12 +62,12 @@ jobs:
 
     - name: Run Nox
       run: |
-        nox --python=${{ matrix.python-version }} --session=test_cookiecutter
+        nox --session=test_cookiecutter
 
     - uses: actions/upload-artifact@v4
       if: always()
       with:
-        name: cookiecutter-${{ matrix.os }}-py${{ matrix.python-version }}
+        name: cookiecutter-ubuntu-latest-py3x
         path: |
           /tmp/tap-*
           /tmp/target-*

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -48,11 +48,17 @@ jobs:
       matrix:
         session: [tests]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version:
+        - "3.8"
+        - "3.9"
+        - "3.10"
+        - "3.11"
+        - "3.12"
+        - "3.13"
         sqlalchemy: ["2"]
         include:
-        - { session: tests,   python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "1" }
-        - { session: doctest, python-version: "3.12", os: "ubuntu-latest", sqlalchemy: "2" }
+        - { session: tests,   python-version: "3.13", os: "ubuntu-latest", sqlalchemy: "1" }
+        - { session: doctest, python-version: "3.13", os: "ubuntu-latest", sqlalchemy: "2" }
         - { session: mypy,    python-version: "3.12",  os: "ubuntu-latest", sqlalchemy: "2" }
         - { session: deps,    python-version: "3.12",  os: "ubuntu-latest", sqlalchemy: "2" }
 
@@ -64,6 +70,7 @@ jobs:
     - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
 
     - name: Upgrade pip
       env:
@@ -152,7 +159,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: '3.x'
 
     - name: Upgrade pip
       env:

diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml
@@ -47,8 +47,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.12"
-        architecture: x64
+        python-version: "3.x"
 
     - name: Bump version
       id: cz-bump

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml
@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 license = "Apache-2.0"
 {%- if cookiecutter.variant != "None (Skip)" %}

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini
@@ -1,7 +1,7 @@
 # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy
 
 [tox]
-envlist = py3{9,10,11,12}
+envlist = py3{9,10,11,12,13}
 requires =
     tox>=4.19
 

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/pyproject.toml
@@ -19,6 +19,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 license = "Apache-2.0"
 {%- if cookiecutter.variant != "None (Skip)" %}

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/tox.ini
@@ -1,7 +1,7 @@
 # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy
 
 [tox]
-envlist = py3{9,10,11,12}
+envlist = py3{9,10,11,12,13}
 requires =
     tox>=4.19
 

diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml b/cookiecutter/target-template/{{cookiecutter.target_id}}/pyproject.toml
@@ -19,6 +19,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 license = "Apache-2.0"
 {%- if cookiecutter.variant != "None (Skip)" %}

diff --git a/cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini b/cookiecutter/target-template/{{cookiecutter.target_id}}/tox.ini
@@ -1,7 +1,7 @@
 # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy
 
 [tox]
-envlist = py3{9,10,11,12}
+envlist = py3{9,10,11,12,13}
 requires =
     tox>=4.19
 

diff --git a/noxfile.py b/noxfile.py
@@ -6,6 +6,7 @@
 import shutil
 import sys
 import tempfile
+import typing as t
 from pathlib import Path
 
 import nox
@@ -23,7 +24,14 @@
 COOKIECUTTER_REPLAY_FILES = list(Path("./e2e-tests/cookiecutters").glob("*.json"))
 
 package = "singer_sdk"
-python_versions = ["3.12", "3.11", "3.10", "3.9", "3.8"]
+python_versions = [
+    "3.13",
+    "3.12",
+    "3.11",
+    "3.10",
+    "3.9",
+    "3.8",
+]
 main_python_version = "3.12"
 locations = "singer_sdk", "tests", "noxfile.py", "docs/conf.py"
 nox.options.sessions = (
@@ -35,7 +43,7 @@
 )
 
 poetry_config = nox.project.load_toml("pyproject.toml")["tool"]["poetry"]
-test_dependencies = poetry_config["group"]["dev"]["dependencies"].keys()
+test_dependencies: dict[str, t.Any] = poetry_config["group"]["dev"]["dependencies"]
 typing_dependencies = poetry_config["group"]["typing"]["dependencies"].keys()
 
 
@@ -53,7 +61,18 @@ def mypy(session: nox.Session) -> None:
 @nox.session(python=python_versions)
 def tests(session: nox.Session) -> None:
     """Execute pytest tests and compute coverage."""
-    session.install(".[faker,jwt,parquet,s3]")
+    extras = [
+        "faker",
+        "jwt",
+        "parquet",
+        "s3",
+    ]
+
+    if session.python == "3.13":
+        # https://github.com/apache/arrow/issues/43519
+        extras.remove("parquet")
+
+    session.install(f".[{','.join(extras)}]")
     session.install(*test_dependencies)
 
     sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION")
@@ -95,7 +114,7 @@ def benches(session: nox.Session) -> None:
     )
 
 
-@nox.session(name="deps", python=python_versions)
+@nox.session(name="deps", python=main_python_version)
 def dependencies(session: nox.Session) -> None:
     """Check issues with dependencies."""
     session.install(".[s3,testing]")

diff --git a/pyproject.toml b/pyproject.toml
@@ -173,6 +173,8 @@ filterwarnings = [
     "ignore:No records were available to test:UserWarning",
     # https://github.com/meltano/sdk/issues/1354
     "ignore:The function singer_sdk.testing.get_standard_tap_tests is deprecated:DeprecationWarning",
+    # TODO: Address this SQLite warning in Python 3.13+
+    "ignore::ResourceWarning",
 ]
 log_cli_level = "INFO"
 markers = [

diff --git a/samples/sample_tap_dummy_json/pyproject.toml b/samples/sample_tap_dummy_json/pyproject.toml
@@ -15,6 +15,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 license = "Apache-2.0"
 

diff --git a/samples/sample_tap_dummy_json/ruff.toml b/samples/sample_tap_dummy_json/ruff.toml
@@ -1,5 +1,5 @@
 src = ["tap_dummyjson"]
-target-version = "py38"
+target-version = "py39"
 
 [lint]
 ignore = [

diff --git a/samples/sample_tap_dummy_json/tox.ini b/samples/sample_tap_dummy_json/tox.ini
@@ -1,7 +1,7 @@
 # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy
 
 [tox]
-envlist = py{38,39,310,311,312}
+envlist = py3{9,10,11,12,13}
 isolated_build = true
 
 [testenv]
@@ -13,7 +13,7 @@ commands =
 [testenv:pytest]
 # Run the python tests.
 # To execute, run `tox -e pytest`
-envlist = py{38,39,310,311,312}
+envlist = py3{8,9,10,11,12,313}
 commands =
     poetry install -v
     poetry run pytest
diff --git a/samples/sample_target_parquet/parquet_target_sink.py b/samples/sample_target_parquet/parquet_target_sink.py
@@ -4,11 +4,14 @@
 
 import typing as t
 
-import pyarrow as pa
-import pyarrow.parquet as pq
-
 from singer_sdk.sinks import BatchSink
 
+try:
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+except ImportError:
+    pass
+
 
 def json_schema_to_arrow(schema: dict[str, t.Any]) -> pa.Schema:
     """Convert a JSON Schema to an Arrow schema.

diff --git a/singer_sdk/about.py b/singer_sdk/about.py
@@ -24,7 +24,7 @@
 
 # Keep these in sync with the supported Python versions in pyproject.toml
 _PY_MIN_VERSION = 8
-_PY_MAX_VERSION = 12
+_PY_MAX_VERSION = 13
 
 
 def _get_min_version(specifiers: SpecifierSet) -> int:

diff --git a/tests/contrib/test_batch_encoder_parquet.py b/tests/contrib/test_batch_encoder_parquet.py
@@ -2,15 +2,23 @@
 
 from __future__ import annotations
 
+import sys
 import typing as t
 
+import pytest
+
 from singer_sdk.contrib.batch_encoder_parquet import ParquetBatcher
 from singer_sdk.helpers._batch import BatchConfig, ParquetEncoding, StorageTarget
 
 if t.TYPE_CHECKING:
     from pathlib import Path
 
 
+@pytest.mark.xfail(
+    sys.version_info >= (3, 13),
+    reason="Parquet not supported on Python 3.13 due to PyArrow incompatibility",
+    strict=True,
+)
 def test_batcher(tmp_path: Path) -> None:
     root = tmp_path.joinpath("batches")
     root.mkdir()
@@ -30,6 +38,11 @@ def test_batcher(tmp_path: Path) -> None:
     assert batches[0][0].endswith(".parquet")
 
 
+@pytest.mark.xfail(
+    sys.version_info >= (3, 13),
+    reason="Parquet not supported on Python 3.13 due to PyArrow incompatibility",
+    strict=True,
+)
 def test_batcher_gzip(tmp_path: Path) -> None:
     root = tmp_path.joinpath("batches")
     root.mkdir()

diff --git a/tests/core/test_about.py b/tests/core/test_about.py
@@ -106,8 +106,8 @@ def test_get_supported_pythons_sdk():
     "specifiers,expected",
     [
         (">=3.7,<3.12", ["3.7", "3.8", "3.9", "3.10", "3.11"]),
-        (">=3.7", ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]),
-        (">3.7", ["3.8", "3.9", "3.10", "3.11", "3.12"]),
+        (">=3.7", ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]),
+        (">3.7", ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]),
         (">3.7,<=3.11", ["3.8", "3.9", "3.10", "3.11"]),
     ],
 )

diff --git a/tests/samples/test_target_parquet.py b/tests/samples/test_target_parquet.py
@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import shutil
+import sys
 import uuid
 from pathlib import Path
 
@@ -23,6 +24,11 @@
 )
 
 
+@pytest.mark.xfail(
+    sys.version_info >= (3, 13),
+    reason="Parquet not supported on Python 3.13 due to PyArrow incompatibility",
+    raises=NameError,
+)
 class TestSampleTargetParquet(StandardTests):
     """Standard Target Tests."""
 

diff --git a/tests/samples/test_target_sqlite.py b/tests/samples/test_target_sqlite.py
@@ -4,6 +4,7 @@
 
 import json
 import sqlite3
+import sys
 import typing as t
 from copy import deepcopy
 from io import StringIO
@@ -367,6 +368,11 @@ def test_sqlite_process_batch_message(
     assert cursor.fetchone()[0] == 4
 
 
+@pytest.mark.xfail(
+    sys.version_info >= (3, 13),
+    reason="Parquet not supported on Python 3.13 due to PyArrow incompatibility",
+    strict=True,
+)
 def test_sqlite_process_batch_parquet(
     sqlite_target_test_config: dict,
     sqlite_sample_target_batch: SQLiteTarget,
-Original file line number
+Diff line change
@@ Expand Up / @@ -32,6 +32,7 @@ body: @@
           description: Version of Python you are using
           options:
             - "NA"
+            - "3.13"
             - "3.12"
             - "3.11"
             - "3.10"
@@ Expand Down @@