From c75e7181fe47745ce7bf73547a2f194942231017 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Mon, 29 Jan 2024 15:39:55 -0800 Subject: [PATCH 01/11] Remove use of six. Remove MorangoMPTT specific handling. --- morango/models/__init__.py | 6 -- morango/models/certificates.py | 7 +- morango/models/core.py | 23 ++---- morango/models/morango_mptt.py | 33 -------- morango/models/utils.py | 11 ++- morango/proquint.py | 5 +- morango/registry.py | 33 +------- morango/sync/syncsession.py | 19 ++--- morango/utils.py | 5 +- .../migrations/0003_auto_20240129_2025.py | 47 ++++++++++++ tests/testapp/facility_profile/models.py | 46 +----------- .../tests/integration/test_morango_mptt.py | 75 ------------------- .../tests/integration/test_serialization.py | 1 - tests/testapp/tests/models/test_core.py | 18 ++--- 14 files changed, 84 insertions(+), 245 deletions(-) delete mode 100644 morango/models/morango_mptt.py create mode 100644 tests/testapp/facility_profile/migrations/0003_auto_20240129_2025.py delete mode 100644 tests/testapp/tests/integration/test_morango_mptt.py diff --git a/morango/models/__init__.py b/morango/models/__init__.py index 4062f16e..1121fadc 100644 --- a/morango/models/__init__.py +++ b/morango/models/__init__.py @@ -21,9 +21,6 @@ from morango.models.fields.crypto import SharedKey from morango.models.fields.uuids import UUIDModelMixin from morango.models.manager import SyncableModelManager -from morango.models.morango_mptt import MorangoMPTTModel -from morango.models.morango_mptt import MorangoMPTTTreeManager -from morango.models.morango_mptt import MorangoTreeQuerySet from morango.models.query import SyncableModelQuerySet from morango.registry import syncable_models @@ -41,9 +38,6 @@ "SyncableModelManager", "SyncableModelQuerySet", "syncable_models", - "MorangoTreeQuerySet", - "MorangoMPTTTreeManager", - "MorangoMPTTModel", "DatabaseIDModel", "InstanceIDModel", "SyncSession", diff --git a/morango/models/certificates.py b/morango/models/certificates.py index 19c8ac19..d6e93713 100644 --- a/morango/models/certificates.py +++ b/morango/models/certificates.py @@ -11,7 +11,6 @@ from django.db import models from django.db import transaction from django.utils import timezone -from django.utils.six import string_types from .fields.crypto import Key from .fields.crypto import PrivateKeyField @@ -195,7 +194,7 @@ def check_certificate(self): def save_certificate_chain(cls, cert_chain, expected_last_id=None): # parse the chain from json if needed - if isinstance(cert_chain, string_types): + if isinstance(cert_chain, str): cert_chain = json.loads(cert_chain) # start from the bottom of the chain @@ -320,7 +319,7 @@ def get_scope(self, params): return Scope(definition=self, params=params) def get_description(self, params): - if isinstance(params, string_types): + if isinstance(params, str): params = json.loads(params) return string.Template(self.description).safe_substitute(params) @@ -328,7 +327,7 @@ def get_description(self, params): class Filter(object): def __init__(self, template, params={}): # ensure params have been deserialized - if isinstance(params, string_types): + if isinstance(params, str): params = json.loads(params) self._template = template self._params = params diff --git a/morango/models/core.py b/morango/models/core.py index 816f33b2..706afde3 100644 --- a/morango/models/core.py +++ b/morango/models/core.py @@ -22,7 +22,6 @@ from django.db.models.expressions import CombinedExpression from django.db.models.fields.related import ForeignKey from django.db.models.functions import Cast -from django.utils import six from django.utils import timezone from django.utils.functional import cached_property @@ -37,7 +36,6 @@ from morango.models.fields.uuids import UUIDModelMixin from morango.models.fsic_utils import remove_redundant_instance_counters from morango.models.manager import SyncableModelManager -from morango.models.morango_mptt import MorangoMPTTModel from morango.models.utils import get_0_4_system_parameters from morango.models.utils import get_0_5_mac_address from morango.models.utils import get_0_5_system_id @@ -353,10 +351,10 @@ def delete_buffers(self): def get_touched_record_ids_for_model(self, model): if isinstance(model, SyncableModel) or ( - isinstance(model, six.class_types) and issubclass(model, SyncableModel) + isinstance(model, type) and issubclass(model, SyncableModel) ): model = model.morango_model_name - _assert(isinstance(model, six.string_types), "Model must resolve to string") + _assert(isinstance(model, str), "Model must resolve to string") return Store.objects.filter( model_name=model, last_transfer_session_id=self.id ).values_list("id", flat=True) @@ -610,7 +608,7 @@ def update_fsics(cls, fsics, sync_filter, v2_format=False): ) else: updated_fsic = {} - for key, value in six.iteritems(fsics): + for key, value in fsics.items(): if key in internal_fsic: # if same instance id, update fsic with larger value if fsics[key] > internal_fsic[key]: @@ -620,7 +618,7 @@ def update_fsics(cls, fsics, sync_filter, v2_format=False): updated_fsic[key] = fsics[key] # load database max counters - for (key, value) in six.iteritems(updated_fsic): + for (key, value) in updated_fsic.items(): for f in sync_filter: DatabaseMaxCounter.objects.update_or_create( instance_id=key, partition=f, defaults={"counter": value} @@ -843,10 +841,8 @@ def delete( with transaction.atomic(): if hard_delete: # set hard deletion for all related models - for model, instances in six.iteritems(collector.data): - if issubclass(model, SyncableModel) or issubclass( - model, MorangoMPTTModel - ): + for model, instances in collector.data.items(): + if issubclass(model, SyncableModel): for obj in instances: obj._update_hard_deleted_models() return collector.delete() @@ -930,13 +926,6 @@ def serialize(self): continue if f.attname in self._morango_internal_fields_not_to_serialize: continue - # case if model is morango mptt - if f.attname in getattr( - self, - "_internal_mptt_fields_not_to_serialize", - "_internal_fields_not_to_serialize", - ): - continue if hasattr(f, "value_from_object_json_compatible"): data[f.attname] = f.value_from_object_json_compatible(self) else: diff --git a/morango/models/morango_mptt.py b/morango/models/morango_mptt.py deleted file mode 100644 index 88abe249..00000000 --- a/morango/models/morango_mptt.py +++ /dev/null @@ -1,33 +0,0 @@ -from mptt import managers -from mptt import models -from mptt import querysets - -from .manager import SyncableModelManager -from .query import SyncableModelQuerySet - - -class MorangoTreeQuerySet(querysets.TreeQuerySet, SyncableModelQuerySet): - pass - - -class MorangoMPTTTreeManager(managers.TreeManager, SyncableModelManager): - def get_queryset(self): - return MorangoTreeQuerySet(self.model, using=self._db) - - def _mptt_update(self, qs=None, **items): - items["update_dirty_bit_to"] = None - return super(MorangoMPTTTreeManager, self)._mptt_update(qs, **items) - - -class MorangoMPTTModel(models.MPTTModel): - """ - Any model that inherits from ``SyncableModel`` that also wants to inherit from ``MPTTModel`` should instead inherit - from ``MorangoMPTTModel``, which modifies some behavior to make it safe for the syncing system. - """ - - _internal_mptt_fields_not_to_serialize = ("lft", "rght", "tree_id", "level") - - objects = MorangoMPTTTreeManager() - - class Meta: - abstract = True diff --git a/morango/models/utils.py b/morango/models/utils.py index e5d26e9d..a221f6d7 100644 --- a/morango/models/utils.py +++ b/morango/models/utils.py @@ -1,5 +1,4 @@ import hashlib -import ifcfg import os import platform import socket @@ -7,10 +6,10 @@ import sys import uuid -from .fields.uuids import sha2_uuid - +import ifcfg from django.conf import settings -from django.utils import six + +from .fields.uuids import sha2_uuid def _get_database_path(): @@ -109,7 +108,7 @@ def _get_android_uuid(): def _do_salted_hash(value): if not value: return "" - if not isinstance(value, six.string_types): + if not isinstance(value, str): value = str(value) try: value = value.encode() @@ -185,7 +184,7 @@ def _get_mac_address_flags(mac): """ See: https://en.wikipedia.org/wiki/MAC_address#Universal_vs._local """ - if isinstance(mac, six.integer_types): + if isinstance(mac, int): mac = _mac_int_to_ether(mac) first_octet = int(mac[:2], base=16) diff --git a/morango/proquint.py b/morango/proquint.py index 0520699f..0280f43b 100644 --- a/morango/proquint.py +++ b/morango/proquint.py @@ -4,7 +4,6 @@ functions. For tighter control over the output, see :class:`HumanHasher`. """ import uuid -from django.utils import six # Copyright (c) 2014 SUNET. All rights reserved. # @@ -59,7 +58,7 @@ def from_int(data): :type data: int :rtype: string """ - if not isinstance(data, six.integer_types): + if not isinstance(data, int): raise TypeError("Input must be integer") res = [] @@ -84,7 +83,7 @@ def to_int(data): :type data: string :rtype: int """ - if not isinstance(data, six.string_types): + if not isinstance(data, str): raise TypeError("Input must be string") res = 0 diff --git a/morango/registry.py b/morango/registry.py index 78924308..ca9ed6a9 100644 --- a/morango/registry.py +++ b/morango/registry.py @@ -7,7 +7,6 @@ from collections import OrderedDict from django.db.models.fields.related import ForeignKey -from django.utils import six from morango.constants import transfer_stages from morango.errors import InvalidMorangoModelConfiguration @@ -111,36 +110,6 @@ def populate(self): # noqa: C901 raise InvalidMorangoModelConfiguration( "Syncing models with more than 1 self referential ForeignKey is not supported." ) - try: - from mptt import models - from morango.models.morango_mptt import ( - MorangoMPTTModel, - MorangoMPTTTreeManager, - MorangoTreeQuerySet, - ) - - # mptt syncable model checks - if issubclass(model, models.MPTTModel): - if not issubclass(model, MorangoMPTTModel): - raise InvalidMorangoModelConfiguration( - "{} that inherits from MPTTModel, should instead inherit from MorangoMPTTModel.".format( - name - ) - ) - if not isinstance(model.objects, MorangoMPTTTreeManager): - raise InvalidMorangoModelConfiguration( - "Manager for {} must inherit from MorangoMPTTTreeManager.".format( - name - ) - ) - if not isinstance(model.objects.none(), MorangoTreeQuerySet): - raise InvalidMorangoModelConfiguration( - "Queryset for {} model must inherit from MorangoTreeQuerySet.".format( - name - ) - ) - except ImportError: - pass # syncable model checks if not isinstance(model.objects, SyncableModelManager): raise InvalidMorangoModelConfiguration( @@ -178,7 +147,7 @@ def populate(self): # noqa: C901 self._insert_model_in_dependency_order(model, profile) # for each profile, create a dict mapping from morango_model_name to model class - for profile, model_list in six.iteritems(self.profile_models): + for profile, model_list in self.profile_models.items(): mapping = OrderedDict() for model in model_list: mapping[model.morango_model_name] = model diff --git a/morango/sync/syncsession.py b/morango/sync/syncsession.py index 1b9d1463..0cc69e4f 100644 --- a/morango/sync/syncsession.py +++ b/morango/sync/syncsession.py @@ -7,12 +7,10 @@ import socket import uuid from io import BytesIO +from urllib.parse import urljoin +from urllib.parse import urlparse from django.utils import timezone -from django.utils.six import iteritems -from django.utils.six import raise_from -from django.utils.six.moves.urllib.parse import urljoin -from django.utils.six.moves.urllib.parse import urlparse from requests.adapters import HTTPAdapter from requests.exceptions import HTTPError from requests.packages.urllib3.util.retry import Retry @@ -299,7 +297,7 @@ def resume_sync_session(self, sync_session_id, chunk_size=None, ignore_existing_ try: self._get_sync_session(sync_session) except HTTPError as e: - raise_from(MorangoResumeSyncError("Failure resuming sync session"), e) + raise MorangoResumeSyncError("Failure resuming sync session") from e # update process id sync_session.process_id = os.getpid() @@ -440,7 +438,7 @@ def _certificate_signing(self, data, userargs, password): # convert user arguments into query str for passing to auth layer if isinstance(userargs, dict): userargs = "&".join( - ["{}={}".format(key, val) for (key, val) in iteritems(userargs)] + ["{}={}".format(key, val) for (key, val) in userargs.items()] ) return self.session.post( self.urlresolve(api_urls.CERTIFICATE), json=data, auth=(userargs, password) @@ -627,12 +625,9 @@ def proceed_to_and_wait_for(self, stage, error_msg=None, callback=None): """ result = self.controller.proceed_to_and_wait_for(stage, callback=callback) if result == transfer_statuses.ERRORED: - raise_from( - MorangoError( - error_msg or "Stage `{}` failed".format(self.context.stage) - ), - self.context.error, - ) + raise MorangoError( + error_msg or "Stage `{}` failed".format(self.context.stage) + ) from self.context.error def initialize(self, sync_filter): """ diff --git a/morango/utils.py b/morango/utils.py index 985c3685..d69038a3 100644 --- a/morango/utils.py +++ b/morango/utils.py @@ -1,14 +1,13 @@ import os -import six from importlib import import_module from django.conf import settings from morango.constants import settings as default_settings from morango.constants.capabilities import ALLOW_CERTIFICATE_PUSHING -from morango.constants.capabilities import GZIP_BUFFER_POST from morango.constants.capabilities import ASYNC_OPERATIONS from morango.constants.capabilities import FSIC_V2_FORMAT +from morango.constants.capabilities import GZIP_BUFFER_POST def do_import(import_string): @@ -32,7 +31,7 @@ class Settings(object): def __getattribute__(self, key): """Coalesces settings with the defaults""" value = getattr(settings, key, getattr(default_settings, key, None)) - if key == "MORANGO_INSTANCE_INFO" and isinstance(value, six.string_types): + if key == "MORANGO_INSTANCE_INFO" and isinstance(value, str): value = dict(do_import(value)) return value diff --git a/tests/testapp/facility_profile/migrations/0003_auto_20240129_2025.py b/tests/testapp/facility_profile/migrations/0003_auto_20240129_2025.py new file mode 100644 index 00000000..6b189635 --- /dev/null +++ b/tests/testapp/facility_profile/migrations/0003_auto_20240129_2025.py @@ -0,0 +1,47 @@ +# Generated by Django 3.2.23 on 2024-01-29 20:25 +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('facility_profile', '0002_auto_20220427_0449'), + ] + + operations = [ + migrations.DeleteModel( + name='ProxyParent', + ), + migrations.DeleteModel( + name='ProxyModel', + ), + migrations.RemoveField( + model_name='facility', + name='level', + ), + migrations.RemoveField( + model_name='facility', + name='lft', + ), + migrations.RemoveField( + model_name='facility', + name='rght', + ), + migrations.RemoveField( + model_name='facility', + name='tree_id', + ), + migrations.AlterField( + model_name='facility', + name='parent', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name='children', + to='facility_profile.facility' + ), + ), + ] diff --git a/tests/testapp/facility_profile/models.py b/tests/testapp/facility_profile/models.py index f2b405dc..ab7ccf56 100644 --- a/tests/testapp/facility_profile/models.py +++ b/tests/testapp/facility_profile/models.py @@ -4,12 +4,10 @@ from django.contrib.auth.models import UserManager from django.db import models from django.utils import timezone -from mptt.models import TreeForeignKey from morango.models.core import SyncableModel from morango.models.fields.uuids import UUIDField from morango.models.manager import SyncableModelManager -from morango.models.morango_mptt import MorangoMPTTModel class FacilityDataSyncableModel(SyncableModel): @@ -23,14 +21,14 @@ class SyncableUserModelManager(SyncableModelManager, UserManager): pass -class Facility(MorangoMPTTModel, FacilityDataSyncableModel): +class Facility(FacilityDataSyncableModel): # Morango syncing settings morango_model_name = "facility" name = models.CharField(max_length=100) now_date = models.DateTimeField(default=timezone.now) - parent = TreeForeignKey('self', null=True, blank=True, related_name='children', db_index=True, on_delete=models.CASCADE) + parent = models.ForeignKey('self', null=True, blank=True, related_name='children', db_index=True, on_delete=models.CASCADE) def calculate_source_id(self, *args, **kwargs): return self.name @@ -100,43 +98,3 @@ def calculate_source_id(self, *args, **kwargs): def calculate_partition(self, *args, **kwargs): return '{user_id}:user:interaction'.format(user_id=self.user.id) - - -class ProxyParent(MorangoMPTTModel): - - kind = models.CharField(max_length=20) - - def save(self, *args, **kwargs): - self._ensure_kind() - super(ProxyParent, self).save(*args, **kwargs) - - def _ensure_kind(self): - if self._KIND: - self.kind = self._KIND - - def calculate_source_id(self, *args, **kwargs): - return None - - def calculate_partition(self, *args, **kwargs): - return '' - - -class ProxyManager(models.Manager): - pass - - -class ProxyModel(ProxyParent): - - morango_model_name = 'proxy' - _KIND = 'proxy' - - objects = ProxyManager() - - class Meta: - proxy = True - - def calculate_source_id(self, *args, **kwargs): - return None - - def calculate_partition(self, *args, **kwargs): - return '' diff --git a/tests/testapp/tests/integration/test_morango_mptt.py b/tests/testapp/tests/integration/test_morango_mptt.py deleted file mode 100644 index d54047b4..00000000 --- a/tests/testapp/tests/integration/test_morango_mptt.py +++ /dev/null @@ -1,75 +0,0 @@ -from django.test import TestCase -from facility_profile.models import Facility -from facility_profile.models import MyUser - -from morango.models.morango_mptt import MorangoMPTTTreeManager -from morango.models.morango_mptt import MorangoTreeQuerySet - - -class MorangoMPTTModelTestCase(TestCase): - - def setUp(self): - Facility.objects.create(name='beans') - - def test_mptt_manager_inheritance(self): - self.assertTrue(isinstance(Facility.objects, MorangoMPTTTreeManager)) - self.assertFalse(isinstance(MyUser.objects, MorangoMPTTTreeManager)) - - def test_mptt_qs_inheritance(self): - self.assertTrue(isinstance(Facility.objects.all(), MorangoTreeQuerySet)) - self.assertFalse(isinstance(MyUser.objects.all(), MorangoTreeQuerySet)) - - def test_mptt_manager_update(self): - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - Facility.objects.update(update_dirty_bit_to=False) - self.assertFalse(Facility.objects.first()._morango_dirty_bit) - Facility.objects.update(update_dirty_bit_to=None) - self.assertFalse(Facility.objects.first()._morango_dirty_bit) - Facility.objects.update(update_dirty_bit_to=True) - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - Facility.objects.update(update_dirty_bit_to=False) - self.assertFalse(Facility.objects.first()._morango_dirty_bit) - Facility.objects.update() - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - Facility.objects.update(update_dirty_bit_to=None) - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - - def test_mptt_qs_update(self): - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - Facility.objects.all().update(update_dirty_bit_to=False) - self.assertFalse(Facility.objects.first()._morango_dirty_bit) - Facility.objects.all().update(update_dirty_bit_to=None) - self.assertFalse(Facility.objects.first()._morango_dirty_bit) - Facility.objects.all().update(update_dirty_bit_to=True) - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - Facility.objects.all().update(update_dirty_bit_to=False) - self.assertFalse(Facility.objects.first()._morango_dirty_bit) - Facility.objects.all().update() - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - Facility.objects.all().update(update_dirty_bit_to=None) - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - - def test_mptt_save(self): - fac = Facility.objects.first() - self.assertTrue(fac._morango_dirty_bit) - fac.save(update_dirty_bit_to=False) - self.assertFalse(Facility.objects.first()._morango_dirty_bit) - fac.save(update_dirty_bit_to=None) - self.assertFalse(Facility.objects.first()._morango_dirty_bit) - fac.save(update_dirty_bit_to=True) - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - fac.save(update_dirty_bit_to=False) - self.assertFalse(Facility.objects.first()._morango_dirty_bit) - fac.save() - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - fac.save(update_dirty_bit_to=None) - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - - def test_new_mptt_update(self): - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - Facility.objects.rebuild() # calls _mptt_update - self.assertTrue(Facility.objects.first()._morango_dirty_bit) - Facility.objects.update(update_dirty_bit_to=False) - self.assertFalse(Facility.objects.first()._morango_dirty_bit) - Facility.objects.rebuild() # calls _mptt_update - self.assertFalse(Facility.objects.first()._morango_dirty_bit) diff --git a/tests/testapp/tests/integration/test_serialization.py b/tests/testapp/tests/integration/test_serialization.py index ce59a67b..e8af1394 100644 --- a/tests/testapp/tests/integration/test_serialization.py +++ b/tests/testapp/tests/integration/test_serialization.py @@ -23,7 +23,6 @@ def test_field_deserialization(self): for f in Facility._meta.concrete_fields: # we remove DateTimeField (for now) from this test because serializing and deserializing loses units of precision if isinstance(f, models.DateTimeField) or \ - f.attname in class_model._internal_mptt_fields_not_to_serialize or \ f.attname in class_model._morango_internal_fields_not_to_serialize: continue self.assertEqual(getattr(self.bob, f.attname), getattr(self.bob_copy, f.attname)) diff --git a/tests/testapp/tests/models/test_core.py b/tests/testapp/tests/models/test_core.py index 9c8a96be..cb7d893b 100644 --- a/tests/testapp/tests/models/test_core.py +++ b/tests/testapp/tests/models/test_core.py @@ -1,21 +1,21 @@ -import factory import uuid -from django.test import TestCase + +import factory from django.test import override_settings +from django.test import TestCase from django.utils import timezone -from django.utils.six import iteritems from facility_profile.models import MyUser from ..helpers import RecordMaxCounterFactory from ..helpers import StoreFactory from morango.constants import transfer_stages from morango.constants import transfer_statuses -from morango.sync.controller import MorangoProfileController from morango.models.certificates import Filter from morango.models.core import DatabaseMaxCounter -from morango.models.core import TransferSession -from morango.models.core import SyncSession from morango.models.core import Store +from morango.models.core import SyncSession +from morango.models.core import TransferSession +from morango.sync.controller import MorangoProfileController class DatabaseMaxCounterFactory(factory.DjangoModelFactory): @@ -118,7 +118,7 @@ def test_update_all_fsics(self): client_fsic = {"a" * 32: 2, "b" * 32: 2, "c" * 32: 2} server_fsic = {"a" * 32: 1, "b" * 32: 1, "c" * 32: 1} self.assertFalse(DatabaseMaxCounter.objects.filter(counter=2).exists()) - for instance_id, counter in iteritems(server_fsic): + for instance_id, counter in server_fsic.items(): DatabaseMaxCounter.objects.create( instance_id=instance_id, counter=counter, partition=self.filter ) @@ -132,7 +132,7 @@ def test_update_some_fsics(self): self.assertFalse( DatabaseMaxCounter.objects.filter(instance_id="e" * 32).exists() ) - for instance_id, counter in iteritems(server_fsic): + for instance_id, counter in server_fsic.items(): DatabaseMaxCounter.objects.create( instance_id=instance_id, counter=counter, partition=self.filter ) @@ -145,7 +145,7 @@ def test_no_fsics_get_updated(self): client_fsic = {"a" * 32: 1, "b" * 32: 1, "c" * 32: 1} server_fsic = {"a" * 32: 2, "b" * 32: 2, "c" * 32: 2} self.assertFalse(DatabaseMaxCounter.objects.filter(counter=1).exists()) - for instance_id, counter in iteritems(server_fsic): + for instance_id, counter in server_fsic.items(): DatabaseMaxCounter.objects.create( instance_id=instance_id, counter=counter, partition=self.filter ) From 8335f61e2d4951774597f916fccd7b40f31d1b4a Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Mon, 29 Jan 2024 15:46:52 -0800 Subject: [PATCH 02/11] Update django dependencies. --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6b96075b..36746d59 100644 --- a/setup.py +++ b/setup.py @@ -23,10 +23,10 @@ 'morango'}, include_package_data=True, install_requires=[ - "django>=1.10,<2", - "django-mptt<0.10.0", + "django>=3,<4", + "django-mptt>0.10.0", "rsa>=3.4.2,<3.5", - "djangorestframework==3.9.1", + "djangorestframework>3.10", "django-ipware>=1.1.6,<1.2", "requests", "ifcfg", From 8a37f84edb13695a05ab89954c1a8b24a6727f42 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Mon, 29 Jan 2024 15:48:46 -0800 Subject: [PATCH 03/11] Remove context argument. --- morango/models/fields/crypto.py | 4 ++-- morango/models/fields/uuids.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/morango/models/fields/crypto.py b/morango/models/fields/crypto.py index 13ac2d2e..65b8626b 100644 --- a/morango/models/fields/crypto.py +++ b/morango/models/fields/crypto.py @@ -344,7 +344,7 @@ def deconstruct(self): class PublicKeyField(RSAKeyBaseField): - def from_db_value(self, value, expression, connection, context): + def from_db_value(self, value, expression, connection): if not value: return None return Key(public_key_string=value) @@ -363,7 +363,7 @@ def get_prep_value(self, value): class PrivateKeyField(RSAKeyBaseField): - def from_db_value(self, value, expression, connection, context): + def from_db_value(self, value, expression, connection): if not value: return None return Key(private_key_string=value) diff --git a/morango/models/fields/uuids.py b/morango/models/fields/uuids.py index d268fc47..4a4f5e40 100644 --- a/morango/models/fields/uuids.py +++ b/morango/models/fields/uuids.py @@ -2,6 +2,7 @@ import uuid from django.db import models + from morango.utils import _assert @@ -41,7 +42,7 @@ def get_db_prep_value(self, value, connection, prepared=False): raise TypeError(self.error_messages["invalid"] % {"value": value}) return value.hex - def from_db_value(self, value, expression, connection, context): + def from_db_value(self, value, expression, connection): return self.to_python(value) def to_python(self, value): From 4c4424a2e3ef349ae9e4d0ef0fc16641f846e28f Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Mon, 29 Jan 2024 15:50:02 -0800 Subject: [PATCH 04/11] Use python functools. Don't use schema editor as context. Update Django middleware name. Update migrations for new mptt version. Set output field on function. --- morango/migrations/0024_auto_20240129_1757.py | 28 +++++++++++++++++++ morango/models/core.py | 2 +- morango/sync/backends/utils.py | 21 +++++++------- tests/testapp/testapp/settings.py | 3 +- 4 files changed, 40 insertions(+), 14 deletions(-) create mode 100644 morango/migrations/0024_auto_20240129_1757.py diff --git a/morango/migrations/0024_auto_20240129_1757.py b/morango/migrations/0024_auto_20240129_1757.py new file mode 100644 index 00000000..99a17a74 --- /dev/null +++ b/morango/migrations/0024_auto_20240129_1757.py @@ -0,0 +1,28 @@ +# Generated by Django 3.2.23 on 2024-01-29 17:57 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ('morango', '0023_add_instance_id_fields'), + ] + + operations = [ + migrations.AlterField( + model_name='certificate', + name='level', + field=models.PositiveIntegerField(editable=False), + ), + migrations.AlterField( + model_name='certificate', + name='lft', + field=models.PositiveIntegerField(editable=False), + ), + migrations.AlterField( + model_name='certificate', + name='rght', + field=models.PositiveIntegerField(editable=False), + ), + ] diff --git a/morango/models/core.py b/morango/models/core.py index 706afde3..715b0c8d 100644 --- a/morango/models/core.py +++ b/morango/models/core.py @@ -418,7 +418,7 @@ def char_ids_list(self): self.annotate(id_cast=Cast("id", TextField())) # remove dashes from char uuid .annotate( - fixed_id=Func(F("id_cast"), Value("-"), Value(""), function="replace") + fixed_id=Func(F("id_cast"), Value("-"), Value(""), function="replace", output_field=TextField()) ) # return as list .values_list("fixed_id", flat=True) diff --git a/morango/sync/backends/utils.py b/morango/sync/backends/utils.py index 696d2c3c..856e71df 100644 --- a/morango/sync/backends/utils.py +++ b/morango/sync/backends/utils.py @@ -1,8 +1,7 @@ import sqlite3 +from functools import lru_cache from importlib import import_module -from django.utils.lru_cache import lru_cache - from morango.errors import MorangoError @@ -110,15 +109,15 @@ def create(self): """ fields = [] params = [] - with self.connection.schema_editor() as schema_editor: - for field in self.fields: - # generates the SQL expression for the table column - field_sql, field_params = schema_editor.column_sql( - self, field, include_default=True - ) - field_sql_name = self.connection.ops.quote_name(field.column) - fields.append("{name} {sql}".format(name=field_sql_name, sql=field_sql)) - params.extend(field_params) + schema_editor = self.connection.schema_editor() + for field in self.fields: + # generates the SQL expression for the table column + field_sql, field_params = schema_editor.column_sql( + self, field, include_default=True + ) + field_sql_name = self.connection.ops.quote_name(field.column) + fields.append("{name} {sql}".format(name=field_sql_name, sql=field_sql)) + params.extend(field_params) with self.connection.cursor() as c: self.backend._create_temporary_table(c, self.sql_name, fields, params) diff --git a/tests/testapp/testapp/settings.py b/tests/testapp/testapp/settings.py index 36012281..7a4ed979 100644 --- a/tests/testapp/testapp/settings.py +++ b/tests/testapp/testapp/settings.py @@ -9,7 +9,6 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.9/ref/settings/ """ - import os # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -48,7 +47,7 @@ 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] From 50b97ce4aedb5b74daf6e955f90e1f4da692f330 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Fri, 1 Mar 2024 14:05:24 -0800 Subject: [PATCH 05/11] Update custom serialization flagging to allow for custom deserialization too. --- morango/models/core.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/morango/models/core.py b/morango/models/core.py index 715b0c8d..34d8429a 100644 --- a/morango/models/core.py +++ b/morango/models/core.py @@ -926,8 +926,8 @@ def serialize(self): continue if f.attname in self._morango_internal_fields_not_to_serialize: continue - if hasattr(f, "value_from_object_json_compatible"): - data[f.attname] = f.value_from_object_json_compatible(self) + if getattr(f, "morango_serialize_to_string", False): + data[f.attname] = f.value_to_string(self) else: data[f.attname] = f.value_from_object(self) return data @@ -938,7 +938,10 @@ def deserialize(cls, dict_model): kwargs = {} for f in cls._meta.concrete_fields: if f.attname in dict_model: - kwargs[f.attname] = dict_model[f.attname] + if getattr(f, "morango_serialize_to_string", False): + kwargs[f.attname] = f.to_python(dict_model[f.attname]) + else: + kwargs[f.attname] = dict_model[f.attname] return cls(**kwargs) @classmethod From 26f0a22c97b796449332383908ff0a96ba5a3713 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 5 Mar 2024 12:17:36 -0800 Subject: [PATCH 06/11] Update syncsession tests to use server in separate process to isolate database. --- tests/testapp/testapp/server2_settings.py | 5 ++ tests/testapp/testapp/settings.py | 7 +- .../tests/integration/test_syncsession.py | 87 ++++++++++++++++--- 3 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 tests/testapp/testapp/server2_settings.py diff --git a/tests/testapp/testapp/server2_settings.py b/tests/testapp/testapp/server2_settings.py new file mode 100644 index 00000000..8aed41a0 --- /dev/null +++ b/tests/testapp/testapp/server2_settings.py @@ -0,0 +1,5 @@ +from .settings import * # noqa + +DATABASES = { + 'default': DATABASES["default2"], # noqa +} diff --git a/tests/testapp/testapp/settings.py b/tests/testapp/testapp/settings.py index 7a4ed979..e5448bea 100644 --- a/tests/testapp/testapp/settings.py +++ b/tests/testapp/testapp/settings.py @@ -24,7 +24,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = ["*"] # Application definition @@ -84,6 +84,11 @@ 'default2': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(BASE_DIR, "testapp2.db"), + 'TEST': { + # Force using a disk based database for testing + # so that we can share it between processes. + 'NAME': os.path.join(BASE_DIR, "testapp2.db"), + } } } diff --git a/tests/testapp/tests/integration/test_syncsession.py b/tests/testapp/tests/integration/test_syncsession.py index 8dda23c0..5e7be2d2 100644 --- a/tests/testapp/tests/integration/test_syncsession.py +++ b/tests/testapp/tests/integration/test_syncsession.py @@ -1,15 +1,22 @@ import contextlib import json +import os +import socket +import subprocess +import sys +import time import mock import pytest +import requests from django.conf import settings -from django.db import connections -from django.test.testcases import LiveServerTestCase +from django.test.testcases import TransactionTestCase from facility_profile.models import InteractionLog from facility_profile.models import MyUser from facility_profile.models import SummaryLog +from requests.exceptions import RequestException from requests.exceptions import Timeout +from testapp.settings import BASE_DIR from ..compat import EnvironmentVarGuard from morango.errors import MorangoError @@ -22,6 +29,7 @@ from morango.models.core import TransferSession from morango.sync.controller import MorangoProfileController + SECOND_TEST_DATABASE = "default2" SECOND_SYSTEM_ID = "default2" @@ -37,18 +45,79 @@ def second_environment(): assert instance1.id != instance2.id +def get_free_tcp_port(): + tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp.bind(("", 0)) + addr, port = tcp.getsockname() + tcp.close() + return port + + +class LiveServer: + def __init__(self): + self.env = os.environ.copy() + self.env["MORANGO_SYSTEM_ID"] = SECOND_SYSTEM_ID + self.port = get_free_tcp_port() + self.host = "127.0.0.1" + self.start() + + @property + def baseurl(self): + return f"http://{self.host}:{self.port}/" + + def start(self): + manage_py_path = os.path.join(BASE_DIR, "manage.py") + self._instance = subprocess.Popen( + [sys.executable, manage_py_path, "runserver", "--nothreading", "--noreload", "--settings", "testapp.server2_settings", f"{self.host}:{self.port}"], + env=self.env, + ) + self._wait_for_server_start() + + def _wait_for_server_start(self, timeout=20): + for i in range(timeout * 2): + try: + resp = requests.get(self.baseurl, timeout=3) + if resp.status_code > 0: + return + except RequestException: + pass + time.sleep(0.5) + + raise Exception("Server did not start within {} seconds".format(timeout)) + + def kill(self): + try: + self._instance.kill() + except OSError: + pass + + @pytest.mark.skipif( getattr(settings, "MORANGO_TEST_POSTGRESQL", False), reason="Not supported" ) -class PushPullClientTestCase(LiveServerTestCase): - multi_db = True +class PushPullClientTestCase(TransactionTestCase): profile = "facilitydata" + databases = ["default", SECOND_TEST_DATABASE] + + @classmethod + def setUpClass(cls): + super(TransactionTestCase, cls).setUpClass() + cls.server = LiveServer() + + @classmethod + def tearDownClass(cls): + # There may not be a 'server' attribute if setUpClass() for some + # reasons has raised an exception. + if hasattr(cls, 'server'): + # Terminate the live server's thread + cls.server.kill() + super(TransactionTestCase, cls).tearDownClass() def setUp(self): super(PushPullClientTestCase, self).setUp() self.profile_controller = MorangoProfileController(self.profile) self.conn = self.profile_controller.create_network_connection( - self.live_server_url + self.server.baseurl ) self.conn.chunk_size = 3 @@ -130,14 +199,6 @@ def _setUpClient(self, primary_partition): ) return self.conn.create_sync_session(client_cert, server_cert) - @classmethod - def _create_server_thread(cls, connections_override): - # override default to point to second environment database - connections_override["default"] = connections["default2"] - return super(PushPullClientTestCase, cls)._create_server_thread( - connections_override - ) - def assertLastActivityUpdate(self, transfer_session=None): """A signal callable that asserts `last_activity_timestamp`s are updated""" if self.last_transfer_activity is not None: From aa034bdd55ef7c04a9afead584ad6715882d669c Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Wed, 6 Mar 2024 11:25:18 -0800 Subject: [PATCH 07/11] Update psycopg2-binary version to latest compatible with Python 3.6. --- requirements/postgres.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/postgres.txt b/requirements/postgres.txt index cd20dd16..632c63e9 100644 --- a/requirements/postgres.txt +++ b/requirements/postgres.txt @@ -1,2 +1,2 @@ -r accelerated.txt -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.9 From 873c4663d37c3874d5e22cc9df29d259bf220cda Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 5 Mar 2024 12:36:39 -0800 Subject: [PATCH 08/11] Update docs dependencies and foreign key introspection. --- docs/conf.py | 2 +- requirements/docs.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index bdfcc995..2bc52256 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -84,7 +84,7 @@ def process_docstring(app, what, name, obj, options, lines): # Add the field's type to the docstring if isinstance(field, models.ForeignKey): - to = field.rel.to + to = field.remote_field.model lines.append(u':type %s: %s to :class:`~%s`' % (field.attname, type(field).__name__, to)) else: lines.append(u':type %s: %s' % (field.attname, type(field).__name__)) diff --git a/requirements/docs.txt b/requirements/docs.txt index 7dc25ef3..e171741e 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -7,8 +7,8 @@ m2r sphinx-notfound-page # for extracting model docs -django==1.11.29 # pyup: >=1.11,<2 -djangorestframework==3.9.1 -django-mptt==0.9.1 +django==3.2.24 +djangorestframework==3.14.0 +django-mptt==0.16.0 rsa==3.4.2 ifcfg==0.21 From 505b2b87d5b9f11bb248cccb5c77e5a9ff3c5656 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 7 Mar 2024 08:27:01 -0800 Subject: [PATCH 09/11] Cleanup unnecessary additional middleware. --- tests/testapp/testapp/settings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/testapp/testapp/settings.py b/tests/testapp/testapp/settings.py index e5448bea..f9a19db4 100644 --- a/tests/testapp/testapp/settings.py +++ b/tests/testapp/testapp/settings.py @@ -47,7 +47,6 @@ 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] From 07f132f74cb1496202e5bb138683fb04a9d9ca3a Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 7 Mar 2024 09:05:33 -0800 Subject: [PATCH 10/11] Update urls.py to remove deprecation warnings. --- morango/urls.py | 6 +++--- tests/testapp/testapp/urls.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/morango/urls.py b/morango/urls.py index 45bad802..01799d66 100644 --- a/morango/urls.py +++ b/morango/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import path -urlpatterns = [url(r"^api/morango/v1/", include("morango.api.urls"))] +urlpatterns = [path("api/morango/v1/", include("morango.api.urls"))] diff --git a/tests/testapp/testapp/urls.py b/tests/testapp/testapp/urls.py index e9ccc940..54f02ad2 100644 --- a/tests/testapp/testapp/urls.py +++ b/tests/testapp/testapp/urls.py @@ -13,9 +13,9 @@ 1. Import the include() function: from django.conf.urls import url, include 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ -from django.conf.urls import include, url -from django.contrib import admin +from django.urls import include +from django.urls import path urlpatterns = [ - url(r'', include('morango.urls')), + path('', include('morango.urls')), ] From d79e77496049f235b18af15899a9e399eb547d1c Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 7 Mar 2024 10:46:22 -0800 Subject: [PATCH 11/11] Upgrade django-ipware. --- morango/api/viewsets.py | 7 ++++++- setup.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/morango/api/viewsets.py b/morango/api/viewsets.py index 2435e6c5..bec91465 100644 --- a/morango/api/viewsets.py +++ b/morango/api/viewsets.py @@ -5,7 +5,7 @@ from django.core.exceptions import ValidationError from django.utils import timezone -from ipware.ip import get_ip +from ipware import get_client_ip from rest_framework import mixins from rest_framework import pagination from rest_framework import response @@ -60,6 +60,11 @@ def controller_signal_logger(context=None): session_controller.signals.connect(controller_signal_logger) +def get_ip(request): + client_ip, _ = get_client_ip(request) + return client_ip + + class CertificateChainViewSet(viewsets.ViewSet): permissions = (permissions.CertificatePushPermissions,) diff --git a/setup.py b/setup.py index 36746d59..d433fc9c 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ "django-mptt>0.10.0", "rsa>=3.4.2,<3.5", "djangorestframework>3.10", - "django-ipware>=1.1.6,<1.2", + "django-ipware==4.0.2", "requests", "ifcfg", ],