From 21dc3954df2f89c1364cdcff25a51e9ee090f42d Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 24 Oct 2024 16:02:00 -0700 Subject: [PATCH] Upgrade dependencies and test matrix for Python 3.13. Add new migration for updated le-utils. Add cgi forward port compatibility patch. --- .github/workflows/morango_integration.yml | 2 +- .github/workflows/tox.yml | 2 +- .../0038_alter_localfile_extension.py | 42 +++++++++++++++ kolibri/utils/compat_cgi.py | 53 +++++++++++++++++++ kolibri/utils/env.py | 16 ++++++ requirements/base.txt | 4 +- setup.py | 3 +- tox.ini | 3 +- 8 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 kolibri/core/content/migrations/0038_alter_localfile_extension.py create mode 100644 kolibri/utils/compat_cgi.py diff --git a/.github/workflows/morango_integration.yml b/.github/workflows/morango_integration.yml index c49eb6faf1e..06a8d95ac1e 100644 --- a/.github/workflows/morango_integration.yml +++ b/.github/workflows/morango_integration.yml @@ -30,7 +30,7 @@ jobs: INTEGRATION_TEST: 'true' strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12', '3.13'] steps: - name: Checkout repository diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index e5223672819..c9ba706a8f9 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -30,7 +30,7 @@ jobs: strategy: max-parallel: 5 matrix: - python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12'] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 diff --git a/kolibri/core/content/migrations/0038_alter_localfile_extension.py b/kolibri/core/content/migrations/0038_alter_localfile_extension.py new file mode 100644 index 00000000000..137a92b12ec --- /dev/null +++ b/kolibri/core/content/migrations/0038_alter_localfile_extension.py @@ -0,0 +1,42 @@ +# Generated by Django 3.2.25 on 2024-10-24 22:56 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("content", "0037_add_bloompub_preset"), + ] + + operations = [ + migrations.AlterField( + model_name="localfile", + name="extension", + field=models.CharField( + blank=True, + choices=[ + ("mp4", "MP4 Video"), + ("webm", "WEBM Video"), + ("vtt", "VTT Subtitle"), + ("mp3", "MP3 Audio"), + ("pdf", "PDF Document"), + ("jpg", "JPG Image"), + ("jpeg", "JPEG Image"), + ("png", "PNG Image"), + ("gif", "GIF Image"), + ("json", "JSON"), + ("svg", "SVG Image"), + ("perseus", "Perseus Exercise"), + ("graphie", "Graphie Exercise"), + ("zip", "HTML5 Zip"), + ("h5p", "H5P"), + ("zim", "ZIM"), + ("epub", "ePub Document"), + ("bloompub", "Bloom Document"), + ("bloomd", "Bloom Document"), + ], + max_length=40, + ), + ), + ] diff --git a/kolibri/utils/compat_cgi.py b/kolibri/utils/compat_cgi.py new file mode 100644 index 00000000000..7336b9835d3 --- /dev/null +++ b/kolibri/utils/compat_cgi.py @@ -0,0 +1,53 @@ +""" +A minimal port of the removed cgi module for use in Python 3.13. +Only imports the specific parts of the module that are used by Django. +Informed by the PR that removed its use in Django: +https://github.com/django/django/pull/15679 +""" + + +def _parseparam(s): + while s[:1] == ";": + s = s[1:] + end = s.find(";") + while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2: + end = s.find(";", end + 1) + if end < 0: + end = len(s) + f = s[:end] + yield f.strip() + s = s[end:] + + +def parse_header(line): + """ + Parse a Content-type like header. + Return the main content-type and a dictionary of options. + """ + parts = _parseparam(";" + line) + key = parts.__next__() + pdict = {} + for p in parts: + i = p.find("=") + if i >= 0: + name = p[:i].strip().lower() + value = p[i + 1 :].strip() + if len(value) >= 2 and value[0] == value[-1] == '"': + value = value[1:-1] + value = value.replace("\\\\", "\\").replace('\\"', '"') + pdict[name] = value + return key, pdict + + +boundary_re = None + + +def valid_boundary(boundary): + # Do this import here to avoid importing dependencies in the env module. + from django.utils.regex_helper import _lazy_re_compile + + global boundary_re + + if boundary_re is None: + boundary_re = _lazy_re_compile(rb"[ -~]{0,200}[!-~]") + return boundary_re.fullmatch(boundary) is not None diff --git a/kolibri/utils/env.py b/kolibri/utils/env.py index 3009c5f75b1..f6aaf076e1c 100644 --- a/kolibri/utils/env.py +++ b/kolibri/utils/env.py @@ -109,6 +109,19 @@ def monkey_patch_distutils(): sys.modules["distutils.version"] = module +def forward_port_cgi_module(): + """ + Forward ports the required parts of the removed cgi module. + This can be removed when we upgrade to a version of Django that is Python 3.13 compatible. + """ + if sys.version_info < (3, 13): + return + from importlib import import_module + + module = import_module("kolibri.utils.compat_cgi") + sys.modules["cgi"] = module + + def set_env(): """ Sets the Kolibri environment for the CLI or other application worker @@ -128,6 +141,9 @@ def set_env(): # Add path for c extensions to sys.path prepend_cext_path(os.path.realpath(os.path.dirname(kolibri_dist.__file__))) + # Depends on Django, so we need to wait until our dist has been registered. + forward_port_cgi_module() + # Set default env for key, value in ENVIRONMENT_VARIABLES.items(): if "default" in value: diff --git a/requirements/base.txt b/requirements/base.txt index e5e29c04110..89a80bb655b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -9,9 +9,9 @@ django-mptt==0.14.0 requests==2.27.1 cheroot==10.0.1 magicbus==4.1.2 -le-utils==0.2.6 +le-utils==0.2.8 jsonfield==3.1.0 -morango==0.8.1 +morango==0.8.2 tzlocal==4.2 pytz==2024.1 python-dateutil==2.9.0.post0 diff --git a/setup.py b/setup.py index 36e42c0bfcd..17136d7108b 100644 --- a/setup.py +++ b/setup.py @@ -100,8 +100,9 @@ def run(self): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: PyPy", ], cmdclass={"install_scripts": gen_windows_batch_files}, - python_requires=">=3.6, <3.13", + python_requires=">=3.6, <3.14", ) diff --git a/tox.ini b/tox.ini index c6364edac97..90fdaae4bd1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{3.6,3.7,3.8,3.9,3.10,3.11,3.12}, postgres +envlist = py{3.6,3.7,3.8,3.9,3.10,3.11,3.12,3.13}, postgres [testenv] usedevelop = True @@ -21,6 +21,7 @@ basepython = py3.10: python3.10 py3.11: python3.11 py3.12: python3.12 + py3.13: python3.13 deps = -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/base.txt