From 9e9d00e5afdef54b1192d408f83e233ce32523b7 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Mon, 11 Sep 2023 08:26:54 -0700 Subject: [PATCH 1/5] Update github action workflows for py2.7 deprecation --- .github/workflows/python-publish.yml | 4 +- .github/workflows/python2-lint.yml | 8 +- .github/workflows/tox.yml | 112 ++++++++++++++++++++------- .gitignore | 3 +- 4 files changed, 91 insertions(+), 36 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 6163b354..5ebde347 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -16,9 +16,9 @@ jobs: deploy: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.9 - name: Install dependencies diff --git a/.github/workflows/python2-lint.yml b/.github/workflows/python2-lint.yml index 668305c0..81ae042b 100644 --- a/.github/workflows/python2-lint.yml +++ b/.github/workflows/python2-lint.yml @@ -19,13 +19,11 @@ jobs: name: Python 2 syntax checking needs: pre_job if: ${{ needs.pre_job.outputs.should_skip != 'true' }} - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 + container: + image: python:2.7.18-buster steps: - uses: actions/checkout@v2 - - name: Set up Python 2.7 - uses: actions/setup-python@v2 - with: - python-version: 2.7 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 80969c8b..87414f62 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -18,67 +18,120 @@ jobs: unit_test: name: Python unit tests needs: pre_job - if: ${{ needs.pre_job.outputs.should_skip != 'true' }} runs-on: ubuntu-18.04 strategy: max-parallel: 5 matrix: - python-version: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9] - + python-version: [3.4, 3.5, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: | python -m pip install --upgrade pip pip install tox - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/cache@v2 with: path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: tox -e py${{ matrix.python-version }} + unit_test27: + name: Python unit tests (2.7) + needs: pre_job + runs-on: ubuntu-20.04 + container: + image: python:2.7.18-buster + steps: + - uses: actions/checkout@v2 + - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + python -m pip install --upgrade pip + pip install tox + - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + uses: actions/cache@v2 + with: + path: ${{ github.workspace }}/.tox/py2.7 + key: ${{ runner.os }}-tox-py2.7-${{ hashFiles('setup.py', 'requirements/*.txt') }} + - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: tox -e py2.7 cryptography: name: Python unit tests + cryptography needs: pre_job - if: ${{ needs.pre_job.outputs.should_skip != 'true' }} runs-on: ubuntu-18.04 strategy: max-parallel: 5 matrix: # only crypto 2.3 seems to work - python-version: [ 2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9 ] + python-version: [ 3.4, 3.5, 3.6, 3.7, 3.8, 3.9 ] crypto-version: [ 2.3 ] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install system dependencies - run: | - sudo apt-get -y -qq update - sudo apt-get install -y openssl libssl-dev - - name: Install tox - run: | - python -m pip install --upgrade pip - pip install tox - - name: tox env cache - uses: actions/cache@v2 - with: - path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} - key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-crypto${{ matrix.crypto-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} - - name: Test with tox - run: tox -e py${{ matrix.python-version }}-cryptography${{ matrix.crypto-version }} + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install system dependencies + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + sudo apt-get -y -qq update + sudo apt-get install -y openssl libssl-dev + - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + python -m pip install --upgrade pip + pip install tox + - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + uses: actions/cache@v2 + with: + path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} + key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-crypto${{ matrix.crypto-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} + - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: tox -e py${{ matrix.python-version }}-cryptography${{ matrix.crypto-version }} + cryptography27: + name: Python unit tests (2.7) + cryptography + needs: pre_job + runs-on: ubuntu-20.04 + container: + image: python:2.7.18-buster + steps: + - uses: actions/checkout@v2 + - name: Install system dependencies + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + apt-get -y -qq update + apt-get install -y openssl libssl-dev + - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: | + python -m pip install --upgrade pip + pip install tox + - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + uses: actions/cache@v2 + with: + path: ${{ github.workspace }}/.tox/py2.7 + key: ${{ runner.os }}-tox-py2.7-crypto3.3-${{ hashFiles('setup.py', 'requirements/*.txt') }} + - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} + run: tox -e py2.7-cryptography2.3 postgres: name: Python postgres unit tests needs: pre_job - if: ${{ needs.pre_job.outputs.should_skip != 'true' }} runs-on: ubuntu-18.04 services: # Label used to access the service container @@ -102,19 +155,23 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python 3.5 for Postgres + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/setup-python@v2 with: python-version: 3.5 - name: Install tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: | python -m pip install --upgrade pip pip install tox - name: tox env cache + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/cache@v2 with: path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} - name: Test with tox + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: tox -e postgres windows: name: Python unit tests on Windows Server @@ -124,7 +181,6 @@ jobs: max-parallel: 5 matrix: python-version: [3.6] - steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index d40ef0ca..8dd2941e 100644 --- a/.gitignore +++ b/.gitignore @@ -85,6 +85,7 @@ venv/ ENV/ .venv Pipfile +.envrc # Spyder project settings .spyderproject @@ -97,4 +98,4 @@ Pipfile *.db -.vscode \ No newline at end of file +.vscode From 2217f6b8f7aba4b8d12dfdbd2138f5277da77dbb Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Mon, 11 Sep 2023 08:30:46 -0700 Subject: [PATCH 2/5] Fix linting issue --- morango/management/commands/cleanupsyncs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/morango/management/commands/cleanupsyncs.py b/morango/management/commands/cleanupsyncs.py index 6cec04b6..42915759 100644 --- a/morango/management/commands/cleanupsyncs.py +++ b/morango/management/commands/cleanupsyncs.py @@ -88,10 +88,10 @@ def handle(self, *args, **options): if options["sync_filter"]: transfer_sessions = transfer_sessions.filter(filter__startswith=options["sync_filter"]) - if options["push"] == True and not options["pull"]: + if options["push"] and not options["pull"]: transfer_sessions = transfer_sessions.filter(push=True) - if options["pull"] == True and not options["push"]: + if options["pull"] and not options["push"]: transfer_sessions = transfer_sessions.filter(push=False) transfer_count = transfer_sessions.count() From 6c733df61a2725876229965c25e1da07a082b393 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Mon, 11 Sep 2023 08:47:48 -0700 Subject: [PATCH 3/5] Python2.7 backports for test.support --- tests/testapp/tests/compat.py | 10 ++++++++++ tests/testapp/tests/helpers.py | 2 +- tests/testapp/tests/integration/test_syncsession.py | 2 +- tests/testapp/tests/sync/test_controller.py | 2 +- tests/testapp/tests/test_api.py | 2 +- tests/testapp/tests/test_uuid_utilities.py | 9 ++++----- tox.ini | 1 + 7 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 tests/testapp/tests/compat.py diff --git a/tests/testapp/tests/compat.py b/tests/testapp/tests/compat.py new file mode 100644 index 00000000..16b1c8a1 --- /dev/null +++ b/tests/testapp/tests/compat.py @@ -0,0 +1,10 @@ +try: + # In the Python 2.7 GH workflows, we have to install backported version + from backports.test.support import EnvironmentVarGuard # noqa F401 +except ImportError: + try: + # For python >2.7 and <3.10 + from test.support import EnvironmentVarGuard # noqa F401 + except ImportError: + # In Python 3.10, this has been moved to test.support.os_helper + from test.support.os_helper import EnvironmentVarGuard # noqa F401 diff --git a/tests/testapp/tests/helpers.py b/tests/testapp/tests/helpers.py index ce7cb832..95d185aa 100644 --- a/tests/testapp/tests/helpers.py +++ b/tests/testapp/tests/helpers.py @@ -3,7 +3,6 @@ """ import json import uuid -from test.support import EnvironmentVarGuard import factory import mock @@ -18,6 +17,7 @@ from facility_profile.models import MyUser from facility_profile.models import SummaryLog +from .compat import EnvironmentVarGuard from morango.api.serializers import BufferSerializer from morango.models.core import AbstractStore from morango.models.core import Buffer diff --git a/tests/testapp/tests/integration/test_syncsession.py b/tests/testapp/tests/integration/test_syncsession.py index 010d422a..4b5d76af 100644 --- a/tests/testapp/tests/integration/test_syncsession.py +++ b/tests/testapp/tests/integration/test_syncsession.py @@ -1,6 +1,5 @@ import contextlib import json -from test.support import EnvironmentVarGuard import mock import pytest @@ -12,6 +11,7 @@ from facility_profile.models import SummaryLog from requests.exceptions import Timeout +from ..compat import EnvironmentVarGuard from morango.errors import MorangoError from morango.models.certificates import Certificate from morango.models.certificates import Filter diff --git a/tests/testapp/tests/sync/test_controller.py b/tests/testapp/tests/sync/test_controller.py index 7fe4df42..b0c16dd3 100644 --- a/tests/testapp/tests/sync/test_controller.py +++ b/tests/testapp/tests/sync/test_controller.py @@ -1,7 +1,6 @@ import contextlib import json import uuid -from test.support import EnvironmentVarGuard import factory import mock @@ -12,6 +11,7 @@ from facility_profile.models import MyUser from facility_profile.models import SummaryLog +from ..compat import EnvironmentVarGuard from ..helpers import serialized_facility_factory from ..helpers import TestSessionContext from morango.constants import transfer_stages diff --git a/tests/testapp/tests/test_api.py b/tests/testapp/tests/test_api.py index a47f94f9..01d7399b 100644 --- a/tests/testapp/tests/test_api.py +++ b/tests/testapp/tests/test_api.py @@ -1,7 +1,6 @@ import json import sys import uuid -from test.support import EnvironmentVarGuard from django.db import connection from django.test.utils import CaptureQueriesContext @@ -13,6 +12,7 @@ from facility_profile.models import MyUser from rest_framework.test import APITestCase as BaseTestCase +from .compat import EnvironmentVarGuard from morango.api.serializers import BufferSerializer from morango.api.serializers import CertificateSerializer from morango.api.serializers import InstanceIDSerializer diff --git a/tests/testapp/tests/test_uuid_utilities.py b/tests/testapp/tests/test_uuid_utilities.py index 7eb53678..74c7c1c4 100644 --- a/tests/testapp/tests/test_uuid_utilities.py +++ b/tests/testapp/tests/test_uuid_utilities.py @@ -1,14 +1,14 @@ import hashlib -import mock import sys import uuid +import mock from django.test import TestCase from facility_profile.models import Facility from facility_profile.models import InteractionLog from facility_profile.models import MyUser -from test.support import EnvironmentVarGuard +from .compat import EnvironmentVarGuard from morango.errors import InvalidMorangoSourceId from morango.models.core import DatabaseIDModel from morango.models.core import InstanceIDModel @@ -157,9 +157,9 @@ def test_consistent_0_5_instance_id(self, *args): env["MORANGO_SYSTEM_ID"] = "magicsysid" DatabaseIDModel.objects.all().update(current=False) - database_id = DatabaseIDModel.objects.create( + DatabaseIDModel.objects.create( id="7fe445b75cea11858c00fb97bdee8878", current=True - ).id + ) self.assertEqual(get_0_5_system_id(), "54940f560a55bbf7d86b") self.assertEqual(get_0_5_mac_address(), "804f4c20d3b2b5a29b95") @@ -197,7 +197,6 @@ def test_envvar_overrides(self, *args): self.assertEqual(instance.id, expected_id) - @mock.patch( "ifcfg.interfaces", return_value={"eth0": {"device": "eth0", "ether": "a0:aa:aa:aa:aa"}}, diff --git a/tox.ini b/tox.ini index 220bf7c4..4138f493 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,7 @@ deps = cryptography1.2: cryptography==1.2.3 cryptography1.8: cryptography==1.8.2 cryptography2.3: cryptography==2.3.1 + py2.7: backports.test.support==0.1.1 commands = sh -c '! tests/testapp/manage.py makemigrations --dry-run --exit --noinput' From fed7ed98aca33e898f24a96edeeeea13cef3963a Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Mon, 11 Sep 2023 15:34:51 -0700 Subject: [PATCH 4/5] Bump ubuntu 18.04 to 20.04, create matrixized jobs for 2.7, 3.4, 3.5 --- .github/workflows/tox.yml | 50 +++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 87414f62..93ba66ff 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -18,11 +18,11 @@ jobs: unit_test: name: Python unit tests needs: pre_job - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: max-parallel: 5 matrix: - python-version: [3.4, 3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -44,12 +44,16 @@ jobs: - name: Test with tox if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: tox -e py${{ matrix.python-version }} - unit_test27: - name: Python unit tests (2.7) + unit_test_deprecated: + name: Python unit tests (deprecated python versions) needs: pre_job runs-on: ubuntu-20.04 + strategy: + max-parallel: 3 + matrix: + python-version: [2.7, 3.4, 3.5] container: - image: python:2.7.18-buster + image: python:${{ matrix.python-version }} steps: - uses: actions/checkout@v2 - name: Install tox @@ -61,20 +65,20 @@ jobs: if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/cache@v2 with: - path: ${{ github.workspace }}/.tox/py2.7 - key: ${{ runner.os }}-tox-py2.7-${{ hashFiles('setup.py', 'requirements/*.txt') }} + path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} + key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} - name: Test with tox if: ${{ needs.pre_job.outputs.should_skip != 'true' }} - run: tox -e py2.7 + run: tox -e py${{ matrix.python-version }} cryptography: name: Python unit tests + cryptography needs: pre_job - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 strategy: max-parallel: 5 matrix: # only crypto 2.3 seems to work - python-version: [ 3.4, 3.5, 3.6, 3.7, 3.8, 3.9 ] + python-version: [ 3.6, 3.7, 3.8, 3.9 ] crypto-version: [ 2.3 ] steps: - uses: actions/checkout@v2 @@ -102,12 +106,16 @@ jobs: - name: Test with tox if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: tox -e py${{ matrix.python-version }}-cryptography${{ matrix.crypto-version }} - cryptography27: - name: Python unit tests (2.7) + cryptography + cryptography_deprecated: + name: Python unit tests (deprecated python versions) + cryptography needs: pre_job runs-on: ubuntu-20.04 + strategy: + max-parallel: 3 + matrix: + python-version: [2.7, 3.4, 3.5] container: - image: python:2.7.18-buster + image: python:${{ matrix.python-version }} steps: - uses: actions/checkout@v2 - name: Install system dependencies @@ -124,15 +132,15 @@ jobs: if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/cache@v2 with: - path: ${{ github.workspace }}/.tox/py2.7 - key: ${{ runner.os }}-tox-py2.7-crypto3.3-${{ hashFiles('setup.py', 'requirements/*.txt') }} + path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} + key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-crypto2.3-${{ hashFiles('setup.py', 'requirements/*.txt') }} - name: Test with tox if: ${{ needs.pre_job.outputs.should_skip != 'true' }} - run: tox -e py2.7-cryptography2.3 + run: tox -e py${{ matrix.python-version }}-cryptography2.3 postgres: name: Python postgres unit tests needs: pre_job - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 services: # Label used to access the service container postgres: @@ -154,11 +162,11 @@ jobs: - 5432:5432 steps: - uses: actions/checkout@v2 - - name: Set up Python 3.5 for Postgres + - name: Set up Python 3.6 for Postgres if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/setup-python@v2 with: - python-version: 3.5 + python-version: 3.6 - name: Install tox if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: | @@ -168,8 +176,8 @@ jobs: if: ${{ needs.pre_job.outputs.should_skip != 'true' }} uses: actions/cache@v2 with: - path: ${{ github.workspace }}/.tox/py${{ matrix.python-version }} - key: ${{ runner.os }}-tox-py${{ matrix.python-version }}-${{ hashFiles('setup.py', 'requirements/*.txt') }} + path: ${{ github.workspace }}/.tox/py3.6 + key: ${{ runner.os }}-tox-py3.6-${{ hashFiles('setup.py', 'requirements/*.txt') }} - name: Test with tox if: ${{ needs.pre_job.outputs.should_skip != 'true' }} run: tox -e postgres From 8fac257d006f50deb12bdea90c94f436cca984e6 Mon Sep 17 00:00:00 2001 From: Blaine Jester Date: Tue, 12 Sep 2023 11:16:56 -0700 Subject: [PATCH 5/5] fix test backports for 3.4 & 3.5, remove 3.4+crypto --- .github/workflows/tox.yml | 6 +++++- tox.ini | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 93ba66ff..68384f99 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -113,7 +113,11 @@ jobs: strategy: max-parallel: 3 matrix: - python-version: [2.7, 3.4, 3.5] + # Packages are no longer available for the distribution that comes with the + # python:3.4 image, so we can't install the necessary dependencies. Eventually, + # this will happen to the others as well. + # EOL: [3.4] + python-version: [2.7, 3.5] container: image: python:${{ matrix.python-version }} steps: diff --git a/tox.ini b/tox.ini index 4138f493..fdc93f9c 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ basepython = py3.7: python3.7 py3.8: python3.8 py3.9: python3.9 - postgres: python3.5 + postgres: python3.6 windows: python3.6 deps = @@ -29,6 +29,8 @@ deps = cryptography1.8: cryptography==1.8.2 cryptography2.3: cryptography==2.3.1 py2.7: backports.test.support==0.1.1 + py3.4: backports.test.support==0.1.1 + py3.5: backports.test.support==0.1.1 commands = sh -c '! tests/testapp/manage.py makemigrations --dry-run --exit --noinput'