Merge pull request #177 from rtibbles/pycrypto_upgrade

Update supported Python versions and cryptography version

.github/workflows/python-publish.yml

@@ -15,7 +15,7 @@ on:
 jobs:
   deploy:
 
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/tox.yml

@@ -19,11 +19,11 @@ jobs:
         name: Python unit tests
         needs: pre_job
         if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-20.04
         strategy:
             max-parallel: 5
             matrix:
-                python-version: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9]
+                python-version: [2.7, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
 
         steps:
             - uses: actions/checkout@v2
@@ -46,13 +46,13 @@ jobs:
         name: Python unit tests + cryptography
         needs: pre_job
         if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-20.04
         strategy:
             max-parallel: 5
             matrix:
-                # only crypto 2.3 seems to work
-                python-version: [ 2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9 ]
-                crypto-version: [ 2.3 ]
+                # only crypto 3.3 seems to work
+                python-version: [ 2.7, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11' ]
+                crypto-version: [ 3.3 ]
 
         steps:
             -   uses: actions/checkout@v2
@@ -79,7 +79,7 @@ jobs:
         name: Python postgres unit tests
         needs: pre_job
         if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
-        runs-on: ubuntu-18.04
+        runs-on: ubuntu-20.04
         services:
             # Label used to access the service container
             postgres:
@@ -101,10 +101,10 @@ jobs:
                     - 5432:5432
         steps:
             - uses: actions/checkout@v2
-            - name: Set up Python 3.5 for Postgres
+            - name: Set up Python 3.9 for Postgres
               uses: actions/setup-python@v2
               with:
-                  python-version: 3.5
+                  python-version: 3.9
             - name: Install tox
               run: |
                   python -m pip install --upgrade pip
@@ -123,7 +123,7 @@ jobs:
         strategy:
             max-parallel: 5
             matrix:
-                python-version: [3.6]
+                python-version: [3.9]
 
         steps:
             - uses: actions/checkout@v2

morango/__init__.py

@@ -3,4 +3,4 @@
 from __future__ import unicode_literals
 
 default_app_config = "morango.apps.MorangoConfig"
-__version__ = "0.6.16"
+__version__ = "0.7.0"

requirements/accelerated.txt

@@ -1,2 +1,2 @@
 M2Crypto==0.37.1
-cryptography==2.3.1
+cryptography==3.3.2

requirements/postgres.txt

@@ -1,2 +1,2 @@
-psycopg2==2.7.4
-cryptography==2.3
+psycopg2-binary==2.8.6
+cryptography==3.3.2

setup.py

@@ -5,7 +5,9 @@
 from __future__ import unicode_literals
 
 import io
-from setuptools import find_packages, setup
+
+from setuptools import find_packages
+from setuptools import setup
 
 import morango
 
@@ -43,8 +45,11 @@
         'Natural Language :: English',
         'Development Status :: 4 - Beta',
         'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
     ],
 )

tests/patch_pytest.py

@@ -0,0 +1,57 @@
+"""
+This script backports this Python 3.10 compatibility fix https://github.com/pytest-dev/pytest/pull/8540
+in order to allow pytest to run in Python 3.10 without upgrading to version 6.2.5 which does not support 2.7
+Copied from:
+https://github.com/learningequality/kolibri/blob/8f9e78572d140a8e70e4d790903fe5acaf4f1a9a/test/patch_pytest.py
+"""
+import os
+import subprocess
+import sys
+
+import pytest
+
+
+def patch():
+    site_packages_dir = os.path.dirname(pytest.__file__)
+
+    patch_file = os.path.join(os.path.dirname(__file__), "pytest_3.10.patch")
+
+    print("Applying patch: " + str(patch_file))
+
+    # -N: insist this is FORWARD patch, don't reverse apply
+    # -p1: strip first path component
+    # -t: batch mode, don't ask questions
+    patch_command = [
+        "patch",
+        "-N",
+        "-p1",
+        "-d",
+        site_packages_dir,
+        "-t",
+        "-i",
+        patch_file,
+    ]
+    print(" ".join(patch_command))
+    try:
+        # Use a dry run to establish whether the patch is already applied.
+        # If we don't check this, the patch may be partially applied (which is bad!)
+        subprocess.check_output(patch_command + ["--dry-run"])
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 1:
+            # Return code 1 means not all hunks could be applied, this usually
+            # means the patch is already applied.
+            print(
+                "Warning: failed to apply patch (exit code 1), "
+                "assuming it is already applied: ",
+                str(patch_file),
+            )
+        else:
+            raise e
+    else:
+        # The dry run worked, so do the real thing
+        subprocess.check_output(patch_command)
+
+
+if __name__ == "__main__":
+    if sys.version_info >= (3, 10):
+        patch()

tests/pytest_3.10.patch

@@ -0,0 +1,38 @@
+diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
+index 4f96b9e8c..1aa5b12de 100644
+--- a/_pytest/assertion/rewrite.py
++++ b/_pytest/assertion/rewrite.py
+@@ -587,10 +587,6 @@ class AssertionRewriter(ast.NodeVisitor):
+             return
+         # Insert some special imports at the top of the module but after any
+         # docstrings and __future__ imports.
+-        aliases = [
+-            ast.alias(py.builtin.builtins.__name__, "@py_builtins"),
+-            ast.alias("_pytest.assertion.rewrite", "@pytest_ar"),
+-        ]
+         doc = getattr(mod, "docstring", None)
+         expect_docstring = doc is None
+         if doc is not None and self.is_rewrite_disabled(doc):
+@@ -617,6 +613,22 @@ class AssertionRewriter(ast.NodeVisitor):
+             pos += 1
+         else:
+             lineno = item.lineno
++        # Now actually insert the special imports.
++        if sys.version_info >= (3, 10):
++            aliases = [
++                ast.alias("builtins", "@py_builtins", lineno=lineno, col_offset=0),
++                ast.alias(
++                    "_pytest.assertion.rewrite",
++                    "@pytest_ar",
++                    lineno=lineno,
++                    col_offset=0,
++                ),
++            ]
++        else:
++            aliases = [
++                ast.alias("builtins", "@py_builtins"),
++                ast.alias("_pytest.assertion.rewrite", "@pytest_ar"),
++            ]
+         imports = [
+             ast.Import([alias], lineno=lineno, col_offset=0) for alias in aliases
+         ]

tests/testapp/manage.py

@@ -2,6 +2,10 @@
 import os
 import sys
 
+import morango # noqa F401
+# Import morango to ensure that we do the monkey patching needed
+# for Django 1.11 to work with Python 3.10+
+
 if __name__ == "__main__":
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "testapp.settings")
 

tests/testapp/testapp/__init__.py

@@ -0,0 +1,76 @@
+import sys
+
+
+def monkey_patch_collections():
+    """
+    Monkey-patching for the collections module is required for Python 3.10
+    and above.
+    Prior to 3.10, the collections module still contained all the entities defined in
+    collections.abc from Python 3.3 onwards. Here we patch those back into main
+    collections module.
+    This can be removed when we upgrade to a version of Django that is Python 3.10 compatible.
+    Copied from:
+    https://github.com/learningequality/kolibri/blob/589dd15aa79e8694aff8754bb34f12384315dbb6/kolibri/utils/compat.py#L90
+    """
+    if sys.version_info < (3, 10):
+        return
+    import collections
+    from collections import abc
+
+    for name in dir(abc):
+        if not hasattr(collections, name):
+            setattr(collections, name, getattr(abc, name))
+
+
+monkey_patch_collections()
+
+
+def monkey_patch_translation():
+    """
+    Monkey-patching for the gettext module is required for Python 3.11
+    and above.
+    Prior to 3.11, the gettext module classes still had the deprecated set_output_charset
+    This can be removed when we upgrade to a version of Django that no longer relies
+    on this deprecated Python 2.7 only call.
+    Copied from:
+    https://github.com/learningequality/kolibri/blob/589dd15aa79e8694aff8754bb34f12384315dbb6/kolibri/utils/compat.py#L109
+    """
+    if sys.version_info < (3, 11):
+        return
+
+    import gettext
+
+    def set_output_charset(*args, **kwargs):
+        pass
+
+    gettext.NullTranslations.set_output_charset = set_output_charset
+
+    original_translation = gettext.translation
+
+    def translation(
+        domain,
+        localedir=None,
+        languages=None,
+        class_=None,
+        fallback=False,
+        codeset=None,
+    ):
+        return original_translation(
+            domain,
+            localedir=localedir,
+            languages=languages,
+            class_=class_,
+            fallback=fallback,
+        )
+
+    gettext.translation = translation
+
+    original_install = gettext.install
+
+    def install(domain, localedir=None, codeset=None, names=None):
+        return original_install(domain, localedir=localedir, names=names)
+
+    gettext.install = install
+
+
+monkey_patch_translation()

tests/testapp/tests/compat.py

@@ -0,0 +1,5 @@
+try:
+    from test.support import EnvironmentVarGuard # noqa F401
+except ImportError:
+    # In Python 3.10, this has been moved to test.support.os_helper
+    from test.support.os_helper import EnvironmentVarGuard # noqa F401

tests/testapp/tests/helpers.py

@@ -3,7 +3,6 @@
 """
 import json
 import uuid
-from test.support import EnvironmentVarGuard
 
 import factory
 import mock
@@ -18,6 +17,7 @@
 from facility_profile.models import MyUser
 from facility_profile.models import SummaryLog
 
+from .compat import EnvironmentVarGuard
 from morango.api.serializers import BufferSerializer
 from morango.models.core import AbstractStore
 from morango.models.core import Buffer

tests/testapp/tests/integration/test_syncsession.py

@@ -1,6 +1,5 @@
 import contextlib
 import json
-from test.support import EnvironmentVarGuard
 
 import mock
 import pytest
@@ -12,6 +11,7 @@
 from facility_profile.models import SummaryLog
 from requests.exceptions import Timeout
 
+from ..compat import EnvironmentVarGuard
 from morango.errors import MorangoError
 from morango.models.certificates import Certificate
 from morango.models.certificates import Filter
@@ -22,7 +22,6 @@
 from morango.models.core import TransferSession
 from morango.sync.controller import MorangoProfileController
 
-
 SECOND_TEST_DATABASE = "default2"
 SECOND_SYSTEM_ID = "default2"
 

tests/testapp/tests/sync/test_controller.py

@@ -1,7 +1,6 @@
 import contextlib
 import json
 import uuid
-from test.support import EnvironmentVarGuard
 
 import factory
 import mock
@@ -12,6 +11,7 @@
 from facility_profile.models import MyUser
 from facility_profile.models import SummaryLog
 
+from ..compat import EnvironmentVarGuard
 from ..helpers import serialized_facility_factory
 from ..helpers import TestSessionContext
 from morango.constants import transfer_stages

tests/testapp/tests/test_api.py

@@ -1,7 +1,6 @@
 import json
 import sys
 import uuid
-from test.support import EnvironmentVarGuard
 
 from django.db import connection
 from django.test.utils import CaptureQueriesContext
@@ -13,6 +12,7 @@
 from facility_profile.models import MyUser
 from rest_framework.test import APITestCase as BaseTestCase
 
+from .compat import EnvironmentVarGuard
 from morango.api.serializers import BufferSerializer
 from morango.api.serializers import CertificateSerializer
 from morango.api.serializers import InstanceIDSerializer