diff --git a/Makefile b/Makefile index afc0b80b064..29dad60d03d 100644 --- a/Makefile +++ b/Makefile @@ -202,7 +202,7 @@ pex: ls dist/*.whl | while read whlfile; do $(MAKE) read-whl-file-version whlfile=$$whlfile; pex $$whlfile --disable-cache -o dist/kolibri-`cat kolibri/VERSION | sed 's/+/_/g'`.pex -m kolibri --python-shebang=/usr/bin/python3; done i18n-extract-backend: - cd kolibri && python -m kolibri manage makemessages -- -l en --ignore 'node_modules/*' --ignore 'kolibri/dist/*' + cd kolibri && python -m kolibri manage makemessages -- -l en --ignore 'node_modules/*' --ignore 'kolibri/dist/*' --all i18n-extract-frontend: yarn run makemessages diff --git a/docs/conf.py b/docs/conf.py index 352a0be0b86..88baa7f7b2e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -80,7 +80,7 @@ def process_docstring(app, what, name, obj, options, lines): # Add the field's type to the docstring if isinstance(field, models.ForeignKey): - to = field.rel.to + to = field.remote_field.model lines.append( u":type %s: %s to :class:`~%s`" % (field.attname, type(field).__name__, to) diff --git a/kolibri/core/api_urls.py b/kolibri/core/api_urls.py index 238eb1bd115..7681b05498f 100644 --- a/kolibri/core/api_urls.py +++ b/kolibri/core/api_urls.py @@ -1,16 +1,16 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path urlpatterns = [ - url(r"^auth/", include("kolibri.core.auth.api_urls")), - url(r"^bookmarks/", include("kolibri.core.bookmarks.api_urls")), - url(r"^content/", include("kolibri.core.content.api_urls")), - url(r"^logger/", include("kolibri.core.logger.api_urls")), - url(r"^tasks/", include("kolibri.core.tasks.api_urls")), - url(r"^exams/", include("kolibri.core.exams.api_urls")), - url(r"^device/", include("kolibri.core.device.api_urls")), - url(r"^lessons/", include("kolibri.core.lessons.api_urls")), - url(r"^discovery/", include("kolibri.core.discovery.api_urls")), - url(r"^notifications/", include("kolibri.core.analytics.api_urls")), - url(r"^public/", include("kolibri.core.public.api_urls")), + re_path(r"^auth/", include("kolibri.core.auth.api_urls")), + re_path(r"^bookmarks/", include("kolibri.core.bookmarks.api_urls")), + re_path(r"^content/", include("kolibri.core.content.api_urls")), + re_path(r"^logger/", include("kolibri.core.logger.api_urls")), + re_path(r"^tasks/", include("kolibri.core.tasks.api_urls")), + re_path(r"^exams/", include("kolibri.core.exams.api_urls")), + re_path(r"^device/", include("kolibri.core.device.api_urls")), + re_path(r"^lessons/", include("kolibri.core.lessons.api_urls")), + re_path(r"^discovery/", include("kolibri.core.discovery.api_urls")), + re_path(r"^notifications/", include("kolibri.core.analytics.api_urls")), + re_path(r"^public/", include("kolibri.core.public.api_urls")), ] diff --git a/kolibri/core/apps.py b/kolibri/core/apps.py index 387c2a2599c..f9cd1eff75b 100644 --- a/kolibri/core/apps.py +++ b/kolibri/core/apps.py @@ -6,7 +6,6 @@ from django.apps import AppConfig from django.conf import settings from django.db.backends.signals import connection_created -from django.db.models.query import F from django.db.utils import DatabaseError from django_filters.filters import UUIDFilter from django_filters.rest_framework.filterset import FilterSet @@ -51,10 +50,6 @@ def ready(self): # Register any django apps that may have kolibri plugin # modules inside them registered_plugins.register_non_plugins(settings.INSTALLED_APPS) - # Fixes issue using OuterRef within Cast() that is patched in later Django version - # Patch from https://github.com/django/django/commit/c412926a2e359afb40738d8177c9f3bef80ee04e - # https://code.djangoproject.com/ticket/29142 - F.relabeled_clone = lambda self, relabels: self @staticmethod def activate_pragmas_per_connection(sender, connection, **kwargs): @@ -116,10 +111,15 @@ def check_redis_settings(): # noqa C901 if we are configured to do so, and if we should, otherwise make some logging noise. """ - from redis.exceptions import ConnectionError - if OPTIONS["Cache"]["CACHE_BACKEND"] != "redis": return + + # Don't import until we've confirmed we're using Redis + # to avoid a hard dependency on Redis + # the options config has already been validated at this point + # so we know that redis is available. + from redis.exceptions import ConnectionError + config_maxmemory = OPTIONS["Cache"]["CACHE_REDIS_MAXMEMORY"] config_maxmemory_policy = OPTIONS["Cache"]["CACHE_REDIS_MAXMEMORY_POLICY"] diff --git a/kolibri/core/auth/api.py b/kolibri/core/auth/api.py index aa3ee5963da..a2a6293f616 100644 --- a/kolibri/core/auth/api.py +++ b/kolibri/core/auth/api.py @@ -192,7 +192,7 @@ class FacilityDatasetViewSet(ValuesViewset): KolibriAuthPermissionsFilter, DjangoFilterBackend, ) - filter_class = FacilityDatasetFilter + filterset_class = FacilityDatasetFilter serializer_class = FacilityDatasetSerializer values = ( @@ -353,6 +353,8 @@ class PublicFacilityUserViewSet(ReadOnlyValuesViewset): } def get_queryset(self): + if self.request.user.is_anonymous: + return FacilityUser.objects.none() facility_id = self.request.query_params.get( "facility_id", self.request.user.facility_id ) @@ -395,9 +397,9 @@ class FacilityUserViewSet(ValuesViewset): ) order_by_field = "username" - queryset = FacilityUser.objects.all() + queryset = FacilityUser.objects.all().order_by(order_by_field) serializer_class = FacilityUserSerializer - filter_class = FacilityUserFilter + filterset_class = FacilityUserFilter search_fields = ("username", "full_name") values = ( @@ -524,7 +526,7 @@ class MembershipViewSet(BulkDeleteMixin, BulkCreateMixin, viewsets.ModelViewSet) filter_backends = (KolibriAuthPermissionsFilter, DjangoFilterBackend) queryset = Membership.objects.all() serializer_class = MembershipSerializer - filter_class = MembershipFilter + filterset_class = MembershipFilter filter_fields = ["user", "collection", "user_ids"] @@ -544,7 +546,7 @@ class RoleViewSet(BulkDeleteMixin, BulkCreateMixin, viewsets.ModelViewSet): filter_backends = (KolibriAuthPermissionsFilter, DjangoFilterBackend) queryset = Role.objects.all() serializer_class = RoleSerializer - filter_class = RoleFilter + filterset_class = RoleFilter filter_fields = ["user", "collection", "kind", "user_ids"] @@ -695,7 +697,7 @@ class ClassroomViewSet(ValuesViewset): filter_backends = (KolibriAuthPermissionsFilter, DjangoFilterBackend) queryset = Classroom.objects.all() serializer_class = ClassroomSerializer - filter_class = ClassroomFilter + filterset_class = ClassroomFilter values = ( "id", diff --git a/kolibri/core/auth/api_urls.py b/kolibri/core/auth/api_urls.py index 8616b82525b..ee7b7a21ee0 100644 --- a/kolibri/core/auth/api_urls.py +++ b/kolibri/core/auth/api_urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import url +from django.urls import re_path from rest_framework import routers from .api import ClassroomViewSet @@ -40,17 +40,17 @@ router.urls + bulk_delete_router.urls + [ - url( + re_path( r"^setnonspecifiedpassword$", SetNonSpecifiedPasswordView.as_view(), name="setnonspecifiedpassword", ), - url( + re_path( r"^usernameavailable$", UsernameAvailableView.as_view(), name="usernameavailable", ), - url( + re_path( r"^ispinvalid/(?P[a-f0-9]{32})$", IsPINValidView.as_view(), name="ispinvalid", diff --git a/kolibri/core/auth/middleware.py b/kolibri/core/auth/middleware.py index 441d1d1f0de..4bea3591a9c 100644 --- a/kolibri/core/auth/middleware.py +++ b/kolibri/core/auth/middleware.py @@ -107,7 +107,7 @@ def __init__(self, get_response): def __call__(self, request): response = self.get_response(request) if response and response.status_code == 401 and request.is_ajax(): - del response["WWW-Authenticate"] + del response.headers["WWW-Authenticate"] return response diff --git a/kolibri/core/auth/models.py b/kolibri/core/auth/models.py index afe805ee295..d0f6ce2d485 100644 --- a/kolibri/core/auth/models.py +++ b/kolibri/core/auth/models.py @@ -31,7 +31,6 @@ from django.db import transaction from django.db.models.query import Q from django.db.utils import IntegrityError -from django.utils.encoding import python_2_unicode_compatible from django.utils.functional import cached_property from morango.models import Certificate from morango.models import SyncableModel @@ -154,7 +153,6 @@ class Meta: abstract = True -@python_2_unicode_compatible class FacilityDataset(FacilityDataSyncableModel): """ ``FacilityDataset`` stores high-level metadata and settings for a particular ``Facility``. @@ -393,45 +391,26 @@ def validate_username(value): validate_username_max_length(value) -class KolibriAbstractBaseUser(AbstractBaseUser): +class KolibriBaseUserMixin: """ - Our custom user type, derived from ``AbstractBaseUser`` as described in the Django docs. - Draws liberally from ``django.contrib.auth.AbstractUser``, except we exclude some fields - we don't care about, like email. - - This model is an abstract model, and is inherited by ``FacilityUser``. + This mixin is inherited by ``KolibriAnonymousUser`` and ``FacilityUser``. + Use a mixin instead of an abstract base class because of difficulties with multiple inheritance and Django's + ``AbstractBaseUser``. """ - class Meta: - abstract = True - - USERNAME_FIELD = "username" - - username = models.CharField( - "username", - max_length=254, - help_text="Required. 254 characters or fewer.", - validators=[validate_username], - ) - full_name = models.CharField("full name", max_length=120, blank=True) - date_joined = DateTimeTzField("date joined", default=local_now, editable=False) - is_staff = False is_superuser = False is_facility_user = False can_manage_content = False - def get_short_name(self): - return self.full_name.split(" ", 1)[0] - @property def session_data(self): """ Data that is added to the session data at login and during session updates. """ raise NotImplementedError( - "Subclasses of KolibriAbstractBaseUser must override the `session_data` property." + "Subclasses of KolibriBaseUserMixin must override the `session_data` property." ) def is_member_of(self, coll): @@ -443,7 +422,7 @@ def is_member_of(self, coll): :rtype: bool """ raise NotImplementedError( - "Subclasses of KolibriAbstractBaseUser must override the `is_member_of` method." + "Subclasses of KolibriBaseUserMixin must override the `is_member_of` method." ) def has_role_for_user(self, kinds, user): @@ -457,7 +436,7 @@ def has_role_for_user(self, kinds, user): :rtype: bool """ raise NotImplementedError( - "Subclasses of KolibriAbstractBaseUser must override the `has_role_for_user` method." + "Subclasses of KolibriBaseUserMixin must override the `has_role_for_user` method." ) def has_role_for_collection(self, kinds, coll): @@ -471,14 +450,14 @@ def has_role_for_collection(self, kinds, coll): :rtype: bool """ raise NotImplementedError( - "Subclasses of KolibriAbstractBaseUser must override the `has_role_for_collection` method." + "Subclasses of KolibriBaseUserMixin must override the `has_role_for_collection` method." ) def can_create_instance(self, obj): """ Checks whether this user (self) has permission to create a particular model instance (obj). - This method should be overridden by classes that inherit from ``KolibriAbstractBaseUser``. + This method should be overridden by classes that inherit from ``KolibriBaseUserMixin``. In general, unless an instance has already been initialized, this method should not be called directly; instead, it should be preferred to call ``can_create``. @@ -488,7 +467,7 @@ def can_create_instance(self, obj): :rtype: bool """ raise NotImplementedError( - "Subclasses of KolibriAbstractBaseUser must override the `can_create_instance` method." + "Subclasses of KolibriBaseUserMixin must override the `can_create_instance` method." ) def can_create(self, Model, data): @@ -526,55 +505,55 @@ def can_read(self, obj): """ Checks whether this user (self) has permission to read a particular model instance (obj). - This method should be overridden by classes that inherit from ``KolibriAbstractBaseUser``. + This method should be overridden by classes that inherit from ``KolibriBaseUserMixin``. :param obj: An instance of a Django model, to check permissions for. :return: ``True`` if this user should have permission to read the object, otherwise ``False``. :rtype: bool """ raise NotImplementedError( - "Subclasses of KolibriAbstractBaseUser must override the `can_read` method." + "Subclasses of KolibriBaseUserMixin must override the `can_read` method." ) def can_update(self, obj): """ Checks whether this user (self) has permission to update a particular model instance (obj). - This method should be overridden by classes that inherit from KolibriAbstractBaseUser. + This method should be overridden by classes that inherit from KolibriBaseUserMixin. :param obj: An instance of a Django model, to check permissions for. :return: ``True`` if this user should have permission to update the object, otherwise ``False``. :rtype: bool """ raise NotImplementedError( - "Subclasses of KolibriAbstractBaseUser must override the `can_update` method." + "Subclasses of KolibriBaseUserMixin must override the `can_update` method." ) def can_delete(self, obj): """ Checks whether this user (self) has permission to delete a particular model instance (obj). - This method should be overridden by classes that inherit from KolibriAbstractBaseUser. + This method should be overridden by classes that inherit from KolibriBaseUserMixin. :param obj: An instance of a Django model, to check permissions for. :return: ``True`` if this user should have permission to delete the object, otherwise ``False``. :rtype: bool """ raise NotImplementedError( - "Subclasses of KolibriAbstractBaseUser must override the `can_delete` method." + "Subclasses of KolibriBaseUserMixin must override the `can_delete` method." ) def has_role_for(self, kinds, obj): """ Helper function that defers to ``has_role_for_user`` or ``has_role_for_collection`` based on the type of object passed in. """ - if isinstance(obj, KolibriAbstractBaseUser): + if isinstance(obj, KolibriBaseUserMixin): return self.has_role_for_user(kinds, obj) elif isinstance(obj, Collection): return self.has_role_for_collection(kinds, obj) else: raise ValueError( - "The `obj` argument to `has_role_for` must be either an instance of KolibriAbstractBaseUser or Collection." + "The `obj` argument to `has_role_for` must be either an instance of KolibriBaseUserMixin or Collection." ) def filter_readable(self, queryset): @@ -585,18 +564,15 @@ def filter_readable(self, queryset): :return: Filtered ``QuerySet`` including only elements that are readable by this user. """ raise NotImplementedError( - "Subclasses of KolibriAbstractBaseUser must override the `can_delete` method." + "Subclasses of KolibriBaseUserMixin must override the `can_delete` method." ) -class KolibriAnonymousUser(AnonymousUser, KolibriAbstractBaseUser): +class KolibriAnonymousUser(AnonymousUser, KolibriBaseUserMixin): """ - Custom anonymous user that also exposes the same interface as KolibriAbstractBaseUser, for consistency. + Custom anonymous user that also exposes the same interface as KolibriBaseUserMixin, for consistency. """ - class Meta: - abstract = True - @property def session_data(self): return { @@ -785,8 +761,7 @@ def validate_role_kinds(kinds): return kinds -@python_2_unicode_compatible -class FacilityUser(KolibriAbstractBaseUser, AbstractFacilityDataModel): +class FacilityUser(AbstractBaseUser, KolibriBaseUserMixin, AbstractFacilityDataModel): """ ``FacilityUser`` is the fundamental object of the auth app. These users represent the main users, and can be associated with a hierarchy of ``Collections`` through ``Memberships`` and ``Roles``, which then serve to help determine permissions. @@ -812,6 +787,17 @@ class FacilityUser(KolibriAbstractBaseUser, AbstractFacilityDataModel): objects = FacilityUserModelManager() + USERNAME_FIELD = "username" + + username = models.CharField( + "username", + max_length=254, + help_text="Required. 254 characters or fewer.", + validators=[validate_username], + ) + full_name = models.CharField("full name", max_length=120, blank=True) + date_joined = DateTimeTzField("date joined", default=local_now, editable=False) + facility = models.ForeignKey("Facility", on_delete=models.CASCADE) is_facility_user = True @@ -834,6 +820,9 @@ class FacilityUser(KolibriAbstractBaseUser, AbstractFacilityDataModel): blank=True, ) + def get_short_name(self): + return self.full_name.split(" ", 1)[0] + @classmethod def deserialize(cls, dict_model): # be defensive against blank passwords, set to `NOT_SPECIFIED` if blank @@ -1043,7 +1032,6 @@ def has_module_perms(self, app_label): return True -@python_2_unicode_compatible class Collection(AbstractFacilityDataModel): """ ``Collections`` are hierarchical groups of ``FacilityUsers``, used for grouping users and making decisions about permissions. @@ -1237,7 +1225,6 @@ def __str__(self): return '"{name}" ({kind})'.format(name=self.name, kind=self.kind) -@python_2_unicode_compatible class Membership(AbstractFacilityDataModel): """ A ``FacilityUser`` can be marked as a member of a ``Collection`` through a ``Membership`` object. Being a member of a @@ -1331,7 +1318,6 @@ def delete(self, **kwargs): return super(Membership, self).delete(**kwargs) -@python_2_unicode_compatible class Role(AbstractFacilityDataModel): """ A ``FacilityUser`` can have a role for a particular ``Collection`` through a ``Role`` object, which also stores @@ -1445,7 +1431,6 @@ def get_queryset(self): ) -@python_2_unicode_compatible class Facility(Collection): # don't require that we have a dataset set during validation, so we're not forced to generate one unnecessarily @@ -1570,7 +1555,6 @@ def __str__(self): return self.name -@python_2_unicode_compatible class Classroom(Collection): morango_model_name = "classroom" @@ -1641,7 +1625,6 @@ def __str__(self): return self.name -@python_2_unicode_compatible class LearnerGroup(Collection): morango_model_name = "learnergroup" @@ -1685,7 +1668,6 @@ def __str__(self): return self.name -@python_2_unicode_compatible class AdHocGroup(Collection): """ An ``AdHocGroup`` is a collection kind that can be used in an assignment diff --git a/kolibri/core/auth/tasks.py b/kolibri/core/auth/tasks.py index caf01bb0b26..1910a925bad 100644 --- a/kolibri/core/auth/tasks.py +++ b/kolibri/core/auth/tasks.py @@ -44,7 +44,7 @@ from kolibri.utils.conf import KOLIBRI_HOME from kolibri.utils.filesystem import mkdirp from kolibri.utils.time_utils import naive_utc_datetime -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ logger = logging.getLogger(__name__) @@ -248,6 +248,16 @@ def validate(self, data): else: facility_id = facility facility_name = data["facility_name"] + kwargs = dict( + chunk_size=200, + noninteractive=True, + ) + if data["command"] == "resumesync": + # Selectively add in the sync_session_id if resuming + # as the sync command will reject the id parameter. + kwargs["id"] = data["sync_session_id"] + else: + kwargs["facility"] = facility_id return { "extra_metadata": dict( facility_id=facility_id, @@ -257,12 +267,7 @@ def validate(self, data): bytes_received=0, ), "facility_id": facility_id, - "kwargs": dict( - chunk_size=200, - noninteractive=True, - facility=facility_id, - sync_session_id=data.get("sync_session_id"), - ), + "kwargs": kwargs, "args": [data["command"]], } diff --git a/kolibri/core/auth/test/migrationtestcase.py b/kolibri/core/auth/test/migrationtestcase.py index 0f24b0bd74e..caa797c185e 100644 --- a/kolibri/core/auth/test/migrationtestcase.py +++ b/kolibri/core/auth/test/migrationtestcase.py @@ -1,13 +1,13 @@ from django.db import connection from django.db.migrations.executor import MigrationExecutor from django.db.migrations.recorder import MigrationRecorder -from django.test import TestCase +from django.test import TransactionTestCase # Modified from https://www.caktusgroup.com/blog/2016/02/02/writing-unit-tests-django-migrations/ -class TestMigrations(TestCase): +class TestMigrations(TransactionTestCase): migrate_from = None migrate_to = None diff --git a/kolibri/core/auth/test/test_api.py b/kolibri/core/auth/test/test_api.py index a4fd8d03b7b..0e99a60b795 100644 --- a/kolibri/core/auth/test/test_api.py +++ b/kolibri/core/auth/test/test_api.py @@ -413,6 +413,8 @@ def test_cannot_create_classroom_no_facility_parent(self): class FacilityAPITestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): provision_device() @@ -892,6 +894,8 @@ def test_cant_add_invalid_extra_demographics_to_facility_user(self): class UserUpdateTestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): provision_device() @@ -1014,6 +1018,8 @@ def test_updating_extra_demographics_previously_set_invalid_value(self): class UserDeleteTestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): provision_device() @@ -1212,6 +1218,8 @@ def test_user_member_of_filter(self): class LoginLogoutTestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): provision_device() @@ -1714,7 +1722,7 @@ def test_facility_admin_can_set_pin_invalid_input(self): def test_facility_admin_can_set_pin_pin_as_none(self): self.client.login(username=self.superuser.username, password=DUMMY_PASSWORD) - response = self.update_pin({"pin_code": None}) + response = self.update_pin({}) self.assertEqual(response.status_code, 400) def test_facility_admin_can_unset_pin(self): @@ -1809,7 +1817,7 @@ def test_facility_admin_can_check_is_pin_valid_empty_payload(self): def test_facility_admin_can_check_is_pin_valid_pin_as_none(self): self.client.login(username=self.superuser.username, password=DUMMY_PASSWORD) self.update_pin({"pin_code": "1234"}) - response = self.is_pin_valid({"pin_code": None}) + response = self.is_pin_valid({}) self.assertEqual(response.status_code, 400) diff --git a/kolibri/core/auth/test/test_auth_tasks.py b/kolibri/core/auth/test/test_auth_tasks.py index 2f4487b1c8e..89d22fa56c8 100644 --- a/kolibri/core/auth/test/test_auth_tasks.py +++ b/kolibri/core/auth/test/test_auth_tasks.py @@ -71,6 +71,8 @@ class dummy_orm_job_data(object): @patch("kolibri.core.tasks.api.job_storage") class FacilityTasksAPITestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): DeviceSettings.objects.create(is_provisioned=True) @@ -152,7 +154,6 @@ def test_startdataportalsync(self, mock_job_storage): facility=self.facility.id, chunk_size=200, noninteractive=True, - sync_session_id=None, ), ) @@ -193,7 +194,6 @@ def test_startdataportalbulksync(self, mock_job_storage): facility=facility2.id, chunk_size=200, noninteractive=True, - sync_session_id=None, ), ) self.assertEqual( @@ -202,7 +202,6 @@ def test_startdataportalbulksync(self, mock_job_storage): facility=facility3.id, chunk_size=200, noninteractive=True, - sync_session_id=None, ), ) @@ -267,7 +266,6 @@ def test_startpeerfacilityimport( no_provision=True, chunk_size=200, noninteractive=True, - sync_session_id=None, ), ) @@ -331,7 +329,6 @@ def test_startpeerfacilitysync( "facility": self.facility.id, "chunk_size": 200, "noninteractive": True, - "sync_session_id": None, }, ) @@ -406,6 +403,8 @@ def test_startdeletefacility__facility_member(self, mock_job_storage): class FacilityTaskHelperTestCase(TestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): cls.device = NetworkLocation.objects.create( @@ -502,7 +501,6 @@ def test_validate_peer_sync_job( kwargs=dict( baseurl="https://some.server.test/", facility=facility_id, - sync_session_id=None, chunk_size=200, noninteractive=True, ), diff --git a/kolibri/core/auth/test/test_bulk_import.py b/kolibri/core/auth/test/test_bulk_import.py index 641bec6b514..a00b76d9c69 100644 --- a/kolibri/core/auth/test/test_bulk_import.py +++ b/kolibri/core/auth/test/test_bulk_import.py @@ -96,6 +96,8 @@ def test_not_empty(): @override_settings(LANGUAGE_CODE="en") class ImportTestCase(TestCase): + databases = "__all__" + def setUp(self): self.data = create_dummy_facility_data( classroom_count=CLASSROOMS, learnergroup_count=1 diff --git a/kolibri/core/auth/test/test_delete_user.py b/kolibri/core/auth/test/test_delete_user.py index 07b3a705090..0afb626c208 100644 --- a/kolibri/core/auth/test/test_delete_user.py +++ b/kolibri/core/auth/test/test_delete_user.py @@ -19,6 +19,8 @@ class UserDeleteTestCase(TestCase): Tests for deleteuser command. """ + databases = "__all__" + @classmethod def setUpTestData(cls): cls.facility = Facility.objects.create() diff --git a/kolibri/core/auth/test/test_deprovisioning.py b/kolibri/core/auth/test/test_deprovisioning.py index 52115a440d0..ba720411bec 100644 --- a/kolibri/core/auth/test/test_deprovisioning.py +++ b/kolibri/core/auth/test/test_deprovisioning.py @@ -29,6 +29,7 @@ class DeprovisionCommandTestCase(TestCase): Tests for the deprovision command. """ + databases = "__all__" fixtures = ["content_test.json"] def setUp(self): diff --git a/kolibri/core/auth/test/test_models.py b/kolibri/core/auth/test/test_models.py index 432ba5593d0..2673dc3202c 100644 --- a/kolibri/core/auth/test/test_models.py +++ b/kolibri/core/auth/test/test_models.py @@ -31,6 +31,8 @@ class CollectionRoleMembershipDeletionTestCase(TestCase): or FacilityUser deletes all associated Roles and Memberships. """ + databases = "__all__" + def setUp(self): self.facility = Facility.objects.create() diff --git a/kolibri/core/auth/test/test_morango_integration.py b/kolibri/core/auth/test/test_morango_integration.py index b6f936f007f..d472619fcc1 100644 --- a/kolibri/core/auth/test/test_morango_integration.py +++ b/kolibri/core/auth/test/test_morango_integration.py @@ -8,6 +8,7 @@ import requests from django.core.management import call_command +from django.db import connections from django.test import TestCase from django.test import TransactionTestCase from django.utils import timezone @@ -78,11 +79,28 @@ def test_deserializing_field(self): self.fail(e.message) +class MultipleServerTestCase(TestCase): + """ + A test case to do special teardown handling to prevent errors from our additional databases. + """ + + @classmethod + def _remove_databases_failures(cls): + for alias in connections: + if alias in cls.databases: + continue + connection = connections[alias] + for name, _ in cls._disallowed_connection_methods: + method = getattr(connection, name) + if hasattr(method, "wrapped"): + setattr(connection, name, method.wrapped) + + @unittest.skipIf( not os.environ.get("INTEGRATION_TEST"), "This test will only be run during integration testing.", ) -class CertificateAuthenticationTestCase(TestCase): +class CertificateAuthenticationTestCase(MultipleServerTestCase): @multiple_kolibri_servers(1) def test_multi_facility_authentication(self, servers): """ @@ -218,7 +236,7 @@ def test_learner_passwordless_authentication(self, servers): not os.environ.get("INTEGRATION_TEST"), "This test will only be run during integration testing.", ) -class EcosystemTestCase(TestCase): +class EcosystemTestCase(MultipleServerTestCase): """ Where possible this test case uses the using kwarg with the db alias in order to save models to the write DB. Unfortunately, because of an internal issue with @@ -589,7 +607,7 @@ def test_chaos_sync(self, servers): not os.environ.get("INTEGRATION_TEST"), "This test will only be run during integration testing.", ) -class EcosystemSingleUserTestCase(TestCase): +class EcosystemSingleUserTestCase(MultipleServerTestCase): @multiple_kolibri_servers(3) def test_single_user_sync(self, servers): @@ -782,7 +800,7 @@ def test_single_user_sync_resumption(self, servers): not os.environ.get("INTEGRATION_TEST"), "This test will only be run during integration testing.", ) -class EcosystemSingleUserAssignmentTestCase(TestCase): +class EcosystemSingleUserAssignmentTestCase(MultipleServerTestCase): @multiple_kolibri_servers(3) def test_single_user_assignment_sync(self, servers): """ @@ -1242,7 +1260,7 @@ def assert_existence(self, server, kind, assignment_id, should_exist=True): not os.environ.get("INTEGRATION_TEST"), "This test will only be run during integration testing.", ) -class SingleUserSyncRegressionsTestCase(TestCase): +class SingleUserSyncRegressionsTestCase(MultipleServerTestCase): @multiple_kolibri_servers(2) def test_facility_user_conflict_syncing_from_tablet(self, servers): self._test_facility_user_conflict(servers, True) diff --git a/kolibri/core/auth/test/test_user_import.py b/kolibri/core/auth/test/test_user_import.py index aee5fb59d62..5eb5d8973d5 100644 --- a/kolibri/core/auth/test/test_user_import.py +++ b/kolibri/core/auth/test/test_user_import.py @@ -149,6 +149,8 @@ class UserImportCommandTestCase(TestCase): Tests for 'kolibri manage importusers' command. """ + databases = "__all__" + @classmethod def setUpClass(self): super(UserImportCommandTestCase, self).setUpClass() diff --git a/kolibri/core/auth/utils/delete.py b/kolibri/core/auth/utils/delete.py index 15ff3f55e12..33ff51daf0a 100644 --- a/kolibri/core/auth/utils/delete.py +++ b/kolibri/core/auth/utils/delete.py @@ -1,7 +1,6 @@ import logging import time -from django.contrib.admin.models import LogEntry from django.db import transaction from django.db.models import Q from django.db.models.signals import post_delete @@ -193,7 +192,6 @@ def _get_users(dataset_id): "User models", querysets=[ LearnerDeviceStatus.objects.filter(dataset_id_filter), - LogEntry.objects.filter(user_id_filter), DevicePermissions.objects.filter(user_id_filter), PingbackNotificationDismissed.objects.filter(user_id_filter), Collection.objects.filter(Q(parent_id__isnull=True) & dataset_id_filter), diff --git a/kolibri/core/auth/utils/migrate.py b/kolibri/core/auth/utils/migrate.py index eadc01b4e8e..5e711ba99aa 100644 --- a/kolibri/core/auth/utils/migrate.py +++ b/kolibri/core/auth/utils/migrate.py @@ -1,7 +1,7 @@ import logging +from django.core.exceptions import FieldDoesNotExist from django.db import connections -from django.db.models import FieldDoesNotExist from morango.registry import syncable_models from morango.sync.backends.utils import calculate_max_sqlite_variables @@ -101,7 +101,7 @@ def _merge_log_data(LogModel): try: field_obj = LogModel._meta.get_field(field) if hasattr(field_obj, "from_db_value"): - value = field_obj.from_db_value(value, None, None, None) + value = field_obj.from_db_value(value, None, None) except FieldDoesNotExist: pass setattr(new_log, field, value) @@ -162,7 +162,7 @@ def _copy_data(Model, id_map, source_data): try: field_obj = Model._meta.get_field(field) if hasattr(field_obj, "from_db_value"): - value = field_obj.from_db_value(value, None, None, None) + value = field_obj.from_db_value(value, None, None) except FieldDoesNotExist: pass setattr(new_obj, field, value) diff --git a/kolibri/core/content/api.py b/kolibri/core/content/api.py index 26df2d674e4..b6685c7187b 100644 --- a/kolibri/core/content/api.py +++ b/kolibri/core/content/api.py @@ -19,7 +19,7 @@ from django.utils.decorators import method_decorator from django.utils.encoding import force_bytes from django.utils.encoding import iri_to_uri -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.cache import cache_page from django.views.decorators.cache import never_cache from django.views.decorators.http import etag @@ -278,7 +278,7 @@ def filter_available(self, queryset, name, value): class BaseChannelMetadataMixin(object): filter_backends = (DjangoFilterBackend,) - filter_class = ChannelMetadataFilter + filterset_class = ChannelMetadataFilter values = ( "author", @@ -582,7 +582,7 @@ class BaseContentNodeMixin(object): """ filter_backends = (DjangoFilterBackend,) - filter_class = ContentNodeFilter + filterset_class = ContentNodeFilter values = ( "id", @@ -1349,7 +1349,7 @@ class ContentNodeBookmarksViewset( KolibriAuthPermissionsFilter, DjangoFilterBackend, ) - filter_class = BookmarkFilter + filterset_class = BookmarkFilter pagination_class = ValuesViewsetLimitOffsetPagination def get_queryset(self): @@ -1391,7 +1391,7 @@ class Meta: class ContentRequestViewset(ReadOnlyValuesViewset, CreateModelMixin): serializer_class = serializers.ContentDownloadRequestSerializer filter_backends = (DjangoFilterBackend,) - filter_class = ContentRequestFilter + filterset_class = ContentRequestFilter pagination_class = OptionalPageNumberPagination permission_classes = [IsAuthenticated] @@ -1480,7 +1480,8 @@ def get_queryset(self): def get_serializer_context(self): context = super(ContentNodeGranularViewset, self).get_serializer_context() - context.update({"channel_stats": self.channel_stats}) + if hasattr(self, "channel_stats"): + context.update({"channel_stats": self.channel_stats}) return context def retrieve(self, request, pk): @@ -1665,7 +1666,7 @@ class UserContentNodeViewset( filter_backends = (DjangoFilterBackend, filters.OrderingFilter) ordering_fields = ["last_interacted"] ordering = ("lft", "id") - filter_class = UserContentNodeFilter + filterset_class = UserContentNodeFilter pagination_class = OptionalPagination def get_queryset(self): @@ -1696,13 +1697,11 @@ def mean(data): return mean -class ContentNodeProgressViewset( - TreeQueryMixin, viewsets.GenericViewSet, ListModelMixin -): +class ContentNodeProgressViewset(TreeQueryMixin, BaseValuesViewset, ListModelMixin): filter_backends = (DjangoFilterBackend, filters.OrderingFilter) ordering_fields = ["last_interacted"] ordering = ("lft", "id") - filter_class = UserContentNodeFilter + filterset_class = UserContentNodeFilter # Use same pagination class as ContentNodeViewset so we can # return identically paginated responses. # The only deviation is that we only return the results @@ -1710,6 +1709,7 @@ class ContentNodeProgressViewset( # that the pagination object generated by the ContentNodeViewset # will be used to make subsequent page requests. pagination_class = OptionalPagination + values = ("content_id", "progress") def get_queryset(self): user = self.request.user @@ -1736,7 +1736,7 @@ def generate_response(self, request, queryset): content_id__in=queryset.exclude(kind=content_kinds.TOPIC).values_list( "content_id", flat=True ), - ).values("content_id", "progress") + ).values(*self.values) ) return Response(logs) diff --git a/kolibri/core/content/api_urls.py b/kolibri/core/content/api_urls.py index a146572aaed..5aaa1b095ac 100644 --- a/kolibri/core/content/api_urls.py +++ b/kolibri/core/content/api_urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from rest_framework import routers from .api import ChannelMetadataViewSet @@ -46,4 +46,4 @@ ) router.register(r"remotechannel", RemoteChannelViewSet, basename="remotechannel") -urlpatterns = [url(r"^", include(router.urls))] +urlpatterns = [re_path(r"^", include(router.urls))] diff --git a/kolibri/core/content/management/commands/content.py b/kolibri/core/content/management/commands/content.py index cd0c5a899ee..244374fa081 100644 --- a/kolibri/core/content/management/commands/content.py +++ b/kolibri/core/content/management/commands/content.py @@ -18,8 +18,7 @@ def add_arguments(self, parser): dest="command", help="The following subcommands are available." ) movedir_subparser = subparsers.add_parser( - name="movedirectory", - cmd=self, + "movedirectory", help="Migrates the content to a specific folder defined by users.", ) movedir_subparser.add_argument( diff --git a/kolibri/core/content/management/commands/importchannel.py b/kolibri/core/content/management/commands/importchannel.py index 59b17e07a94..7cd28e599e2 100644 --- a/kolibri/core/content/management/commands/importchannel.py +++ b/kolibri/core/content/management/commands/importchannel.py @@ -48,8 +48,7 @@ def add_arguments(self, parser): ) network_subparser = subparsers.add_parser( - name="network", - cmd=self, + "network", help="Download the given channel through the network.", ) network_subparser.add_argument( @@ -80,7 +79,7 @@ def add_arguments(self, parser): ) local_subparser = subparsers.add_parser( - name="disk", cmd=self, help="Copy the content from the given folder." + "disk", help="Copy the content from the given folder." ) local_subparser.add_argument( "channel_id", diff --git a/kolibri/core/content/management/commands/importcontent.py b/kolibri/core/content/management/commands/importcontent.py index e01c8c8faf1..6124c97288d 100644 --- a/kolibri/core/content/management/commands/importcontent.py +++ b/kolibri/core/content/management/commands/importcontent.py @@ -128,8 +128,7 @@ def add_arguments(self, parser): # parser object with its own thing, hence why we need to add cmd. See # http://stackoverflow.com/questions/36706220/is-it-possible-to-create-subparsers-in-a-django-management-command network_subparser = subparsers.add_parser( - name="network", - cmd=self, + "network", help="Download the given channel through the network.", ) network_subparser.add_argument("channel_id", type=str) @@ -158,7 +157,7 @@ def add_arguments(self, parser): ) disk_subparser = subparsers.add_parser( - name="disk", cmd=self, help="Copy the content from the given folder." + "disk", help="Copy the content from the given folder." ) disk_subparser.add_argument("channel_id", type=str) disk_subparser.add_argument("directory", type=str, nargs="?") diff --git a/kolibri/core/content/migrations/0036_null_boolean_and_mptt.py b/kolibri/core/content/migrations/0036_null_boolean_and_mptt.py new file mode 100644 index 00000000000..4929a571e7e --- /dev/null +++ b/kolibri/core/content/migrations/0036_null_boolean_and_mptt.py @@ -0,0 +1,52 @@ +# Generated by Django 3.2.25 on 2024-03-22 20:41 +from django.db import migrations +from django.db import models + +import kolibri.core.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("content", "0035_add_imscp_preset"), + ] + + operations = [ + migrations.AlterField( + model_name="channelmetadata", + name="partial", + field=models.BooleanField(default=False, null=True), + ), + migrations.AlterField( + model_name="channelmetadata", + name="public", + field=models.BooleanField(null=True), + ), + migrations.AlterField( + model_name="contentnode", + name="admin_imported", + field=models.BooleanField(null=True), + ), + migrations.AlterField( + model_name="contentnode", + name="ancestors", + field=kolibri.core.fields.JSONField( + blank=True, default=[], load_kwargs={"strict": False}, null=True + ), + ), + migrations.AlterField( + model_name="contentnode", + name="level", + field=models.PositiveIntegerField(editable=False), + ), + migrations.AlterField( + model_name="contentnode", + name="lft", + field=models.PositiveIntegerField(editable=False), + ), + migrations.AlterField( + model_name="contentnode", + name="rght", + field=models.PositiveIntegerField(editable=False), + ), + ] diff --git a/kolibri/core/content/models.py b/kolibri/core/content/models.py index 7eea4a65a04..a400d5600b7 100644 --- a/kolibri/core/content/models.py +++ b/kolibri/core/content/models.py @@ -33,7 +33,6 @@ from django.db.models import Min from django.db.models import OuterRef from django.db.models import QuerySet -from django.utils.encoding import python_2_unicode_compatible from le_utils.constants import content_kinds from le_utils.constants import format_presets from morango.models.fields import UUIDField @@ -58,7 +57,6 @@ PRESET_LOOKUP = dict(format_presets.choices) -@python_2_unicode_compatible class ContentTag(base_models.ContentTag): def __str__(self): return self.tag_name @@ -185,7 +183,6 @@ def treeify(data, cursor=1, level=0): return stack -@python_2_unicode_compatible class ContentNode(base_models.ContentNode): """ The primary object type in a content database. Defines the properties that are shared @@ -215,9 +212,9 @@ class ContentNode(base_models.ContentNode): # whether a device super admin, or via initial configuration. # These nodes will not be subject to automatic garbage collection # to manage space. - # Set as a NullBooleanField to limit migration time in creating the new column, + # Set as a nullable BooleanField to limit migration time in creating the new column, # needs a subsequent Kolibri upgrade step to backfill these values. - admin_imported = models.NullBooleanField() + admin_imported = models.BooleanField(null=True) objects = ContentNodeManager() @@ -248,7 +245,6 @@ def get_descendant_content_ids(self): field.contribute_to_class(ContentNode, field_name) -@python_2_unicode_compatible class Language(base_models.Language): def __str__(self): return self.lang_name or "" @@ -316,7 +312,6 @@ def get_unused_files(self): ) -@python_2_unicode_compatible class LocalFile(base_models.LocalFile): """ The bottom layer of the contentDB schema, defines the local state of files on the device storage. @@ -375,7 +370,6 @@ class ChannelMetadataQueryset(QuerySet, FilterByUUIDQuerysetMixin): BATCH_SIZE = 1000 -@python_2_unicode_compatible class ChannelMetadata(base_models.ChannelMetadata): """ Holds metadata about all existing content databases that exist locally. @@ -388,10 +382,10 @@ class ChannelMetadata(base_models.ChannelMetadata): "Language", related_name="channels", verbose_name="languages", blank=True ) order = models.PositiveIntegerField(default=0, null=True, blank=True) - public = models.NullBooleanField() + public = models.BooleanField(null=True) # Has only a subset of this channel's metadata been imported? # Use a null boolean field to avoid issues during metadata import - partial = models.NullBooleanField(default=False) + partial = models.BooleanField(null=True, default=False) objects = ChannelMetadataQueryset.as_manager() diff --git a/kolibri/core/content/public_api.py b/kolibri/core/content/public_api.py index cb679cca3f2..7b63db14f47 100644 --- a/kolibri/core/content/public_api.py +++ b/kolibri/core/content/public_api.py @@ -3,8 +3,8 @@ from django.db import connection from django.db.models import Q from django.http import HttpResponseBadRequest -from rest_framework.generics import get_object_or_404 from rest_framework.response import Response +from rest_framework.serializers import Serializer from rest_framework.viewsets import GenericViewSet from kolibri.core.content import models @@ -14,6 +14,14 @@ class ImportMetadataViewset(GenericViewSet): + queryset = models.ContentNode.objects.all() + + def get_serializer_class(self): + """ + Add this purely to avoid warnings from DRF YASG schema generation. + """ + return Serializer + default_content_schema = CONTENT_SCHEMA_VERSION min_content_schema = MIN_CONTENT_SCHEMA_VERSION @@ -64,7 +72,7 @@ def retrieve(self, request, pk=None): # Get the model for the target node here - we do this so that we trigger a 404 immediately if the node # does not exist. - node = get_object_or_404(models.ContentNode.objects.all(), pk=pk) + node = self.get_object() nodes = node.get_ancestors(include_self=True) diff --git a/kolibri/core/content/tasks.py b/kolibri/core/content/tasks.py index fa7be917d11..1341614b4ca 100644 --- a/kolibri/core/content/tasks.py +++ b/kolibri/core/content/tasks.py @@ -45,7 +45,7 @@ from kolibri.core.tasks.validation import JobValidator from kolibri.core.utils.urls import reverse_remote from kolibri.utils import conf -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ from kolibri.utils.version import version_matches_range QUEUE = "content" @@ -225,7 +225,6 @@ def remotechannelimport(channel_id, baseurl=None, peer_id=None): "network", channel_id, baseurl=baseurl, - peer_id=peer_id, ) @@ -636,7 +635,6 @@ def diskchannelimport( "disk", channel_id, drive.datafolder, - drive_id=drive_id, ) diff --git a/kolibri/core/content/test/test_annotation.py b/kolibri/core/content/test/test_annotation.py index 462a2de5066..be7af247a4a 100644 --- a/kolibri/core/content/test/test_annotation.py +++ b/kolibri/core/content/test/test_annotation.py @@ -41,6 +41,7 @@ def get_engine(connection_string): @patch("kolibri.core.content.utils.sqlalchemybridge.get_engine", new=get_engine) class SetContentNodesInvisibleTestCase(TransactionTestCase): + databases = "__all__" fixtures = ["content_test.json"] def test_all_leaves(self): diff --git a/kolibri/core/content/test/test_channel_upgrade.py b/kolibri/core/content/test/test_channel_upgrade.py index e78d2b292c2..5e5c3439414 100644 --- a/kolibri/core/content/test/test_channel_upgrade.py +++ b/kolibri/core/content/test/test_channel_upgrade.py @@ -1,4 +1,5 @@ import copy +import json import logging import random import tempfile @@ -314,9 +315,14 @@ def get_resource_localfiles(self, ids): @property def data(self): + contentnode_list = [] + for node in self.nodes.values(): + node["ancestors"] = json.dumps(node["ancestors"]) + contentnode_list.append(node) + return { "content_channel": [self.channel], - "content_contentnode": list(self.nodes.values()), + "content_contentnode": contentnode_list, "content_file": list(self.files.values()), "content_localfile": list(self.localfiles.values()), } diff --git a/kolibri/core/content/test/test_content_app.py b/kolibri/core/content/test/test_content_app.py index e3b99cef2ad..21ffc5711b8 100644 --- a/kolibri/core/content/test/test_content_app.py +++ b/kolibri/core/content/test/test_content_app.py @@ -42,6 +42,8 @@ class ContentNodeTestBase(object): Basecase for content metadata methods """ + databases = "__all__" + def test_get_prerequisites_for(self): """ test the directional characteristic of prerequisite relationship @@ -396,7 +398,7 @@ def test_contentnode_etag(self): self.assertEqual(response.status_code, 200) self.assertNotIn("HTTP_IF_NONE_MATCH", response.request) self.assertEqual(len(response.data), expected_len) - self.assertEqual(response["ETag"], expected_etag) + self.assertEqual(response.headers["ETag"], expected_etag) self.assertIn(url, self.client_cache) cached_response = self.client_cache[url] self.assertEqual(len(cached_response.data), expected_len) @@ -408,7 +410,7 @@ def test_contentnode_etag(self): self.assertIn("HTTP_IF_NONE_MATCH", response.request) self.assertEqual(response.request["HTTP_IF_NONE_MATCH"], expected_etag) self.assertEqual(response.content, b"") - self.assertEqual(response["ETag"], '"{}"'.format(cache_key)) + self.assertEqual(response.headers["ETag"], '"{}"'.format(cache_key)) # Update the content cache key to get a new response. time.sleep(0.01) @@ -421,7 +423,7 @@ def test_contentnode_etag(self): self.assertIn("HTTP_IF_NONE_MATCH", response.request) self.assertEqual(response.request["HTTP_IF_NONE_MATCH"], old_expected_etag) self.assertEqual(len(response.data), expected_len) - self.assertEqual(response["ETag"], '"{}"'.format(cache_key)) + self.assertEqual(response.headers["ETag"], '"{}"'.format(cache_key)) old_cached_response = cached_response cached_response = self.client_cache[url] self.assertEqual(len(cached_response.data), expected_len) @@ -434,7 +436,7 @@ def test_contentnode_etag(self): self.assertIn("HTTP_IF_NONE_MATCH", response.request) self.assertEqual(response.request["HTTP_IF_NONE_MATCH"], expected_etag) self.assertEqual(response.content, b"") - self.assertEqual(response["ETag"], '"{}"'.format(cache_key)) + self.assertEqual(response.headers["ETag"], '"{}"'.format(cache_key)) @unittest.skipIf( getattr(settings, "DATABASES")["default"]["ENGINE"] @@ -1214,8 +1216,7 @@ def test_contentnode_bad_content_id(self): reverse("kolibri:core:contentnode-list"), data={"content_id": "this is not a uuid"}, ) - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.data), 0) + self.assertEqual(response.status_code, 400) def test_contentnode_parent(self): parent = content.ContentNode.objects.get(title="c2") @@ -1585,7 +1586,7 @@ def test_resume_zero_cache(self): response = self.client.get( reverse("kolibri:core:usercontentnode-list"), data={"resume": True} ) - self.assertEqual(response["Cache-Control"], "max-age=0") + self.assertEqual(response.headers["Cache-Control"], "max-age=0") def test_next_steps_prereq(self): facility = Facility.objects.create(name="MyFac") @@ -1628,7 +1629,7 @@ def test_next_steps_prereq_zero_cache(self): response = self.client.get( reverse("kolibri:core:usercontentnode-list"), data={"next_steps": True} ) - self.assertEqual(response["Cache-Control"], "max-age=0") + self.assertEqual(response.headers["Cache-Control"], "max-age=0") def test_next_steps_prereq_in_progress(self): facility = Facility.objects.create(name="MyFac") diff --git a/kolibri/core/content/test/test_delete_content.py b/kolibri/core/content/test/test_delete_content.py index f920c0c82c7..a6155abc80f 100644 --- a/kolibri/core/content/test/test_delete_content.py +++ b/kolibri/core/content/test/test_delete_content.py @@ -25,6 +25,8 @@ def get_engine(connection_string): @patch("kolibri.core.content.utils.sqlalchemybridge.get_engine", new=get_engine) class UnavailableContentDeletion(TransactionTestCase): + databases = "__all__" + def setUp(self): super(UnavailableContentDeletion, self).setUp() @@ -110,6 +112,7 @@ class DeleteContentTestCase(TransactionTestCase): Testcase for delete content management command """ + databases = "__all__" fixtures = ["content_test.json"] def _get_node_ids(self): diff --git a/kolibri/core/content/test/test_deletechannel.py b/kolibri/core/content/test/test_deletechannel.py index dc0db9bd420..e653d64adf4 100644 --- a/kolibri/core/content/test/test_deletechannel.py +++ b/kolibri/core/content/test/test_deletechannel.py @@ -19,6 +19,7 @@ class DeleteChannelTestCase(TransactionTestCase): Testcase for delete channel management command """ + databases = "__all__" fixtures = ["content_test.json"] the_channel_id = "6199dde695db4ee4ab392222d5af1e5c" diff --git a/kolibri/core/content/test/test_file_availability.py b/kolibri/core/content/test/test_file_availability.py index f585c8dc9d6..dc523c425ba 100644 --- a/kolibri/core/content/test/test_file_availability.py +++ b/kolibri/core/content/test/test_file_availability.py @@ -30,6 +30,7 @@ def get_engine(connection_string): @patch("kolibri.core.content.utils.sqlalchemybridge.get_engine", new=get_engine) class LocalFileByDisk(TransactionTestCase): + databases = "__all__" fixtures = ["content_test.json"] @@ -130,6 +131,7 @@ def tearDown(self): @patch("kolibri.core.content.utils.sqlalchemybridge.get_engine", new=get_engine) class LocalFileRemote(TransactionTestCase): + databases = "__all__" fixtures = ["content_test.json"] diff --git a/kolibri/core/content/test/test_public_api.py b/kolibri/core/content/test/test_public_api.py index 62d5d7b828d..790658d630c 100644 --- a/kolibri/core/content/test/test_public_api.py +++ b/kolibri/core/content/test/test_public_api.py @@ -57,7 +57,7 @@ def _assert_data(self, Model, queryset): if field.column in field_names: value = response_data[field.column] if hasattr(field, "from_db_value"): - value = field.from_db_value(value, None, connection, None) + value = field.from_db_value(value, None, connection) self.assertEqual(value, getattr(obj, field.column)) def test_import_metadata_nodes(self): diff --git a/kolibri/core/content/test/test_tasks.py b/kolibri/core/content/test/test_tasks.py index a051cfc76a7..b5f356d1d4a 100644 --- a/kolibri/core/content/test/test_tasks.py +++ b/kolibri/core/content/test/test_tasks.py @@ -117,6 +117,8 @@ def test_returns_right_data(self): class ValidateRemoteImportTaskTestCase(TestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): cls.facility = Facility.objects.create(name="pytest_facility") diff --git a/kolibri/core/content/test/test_upgrade.py b/kolibri/core/content/test/test_upgrade.py index 9f1d60b83d1..2992f8bef87 100644 --- a/kolibri/core/content/test/test_upgrade.py +++ b/kolibri/core/content/test/test_upgrade.py @@ -28,6 +28,7 @@ def get_engine(connection_string): @patch("kolibri.core.content.upgrade.import_channel_from_local_db") class FixMultipleTreesWithId1TestCase(TransactionTestCase): + databases = "__all__" fixtures = ["content_test.json"] diff --git a/kolibri/core/content/test/test_zipcontent.py b/kolibri/core/content/test/test_zipcontent.py index 209c98ee7ac..becf0d5e993 100644 --- a/kolibri/core/content/test/test_zipcontent.py +++ b/kolibri/core/content/test/test_zipcontent.py @@ -245,7 +245,7 @@ def test_request_for_html_body_script_return_correct_length_header(self): ) ) file_size = len(expected_content) - self.assertEqual(int(response["Content-Length"]), file_size) + self.assertEqual(int(response.headers["Content-Length"]), file_size) def test_request_for_html_empty_html(self): response = self._get_file(self.empty_html_name) diff --git a/kolibri/core/content/test/utils/test_assignment.py b/kolibri/core/content/test/utils/test_assignment.py index 127806f4859..60b0d74c4e2 100644 --- a/kolibri/core/content/test/utils/test_assignment.py +++ b/kolibri/core/content/test/utils/test_assignment.py @@ -20,6 +20,8 @@ class ContentAssignmentManagerTestCase(TestCase): + databases = "__all__" + def setUp(self): super(ContentAssignmentManagerTestCase, self).setUp() @@ -288,6 +290,8 @@ def test_get_assignments__one_to_many(self): class ContentAssignmentManagerIntegrationTestCase(TestCase): + databases = "__all__" + @classmethod def setUpClass(cls): super(ContentAssignmentManagerIntegrationTestCase, cls).setUpClass() diff --git a/kolibri/core/content/test/utils/test_content_request.py b/kolibri/core/content/test/utils/test_content_request.py index 0c403c511db..8585995935c 100644 --- a/kolibri/core/content/test/utils/test_content_request.py +++ b/kolibri/core/content/test/utils/test_content_request.py @@ -44,7 +44,7 @@ def _facility(dataset_id=None): class BaseTestCase(TestCase): - multi_db = True + databases = "__all__" def _create_sync_and_network_location( self, sync_overrides=None, location_overrides=None diff --git a/kolibri/core/content/urls.py b/kolibri/core/content/urls.py index a912644f3b8..8f0d5115daa 100644 --- a/kolibri/core/content/urls.py +++ b/kolibri/core/content/urls.py @@ -1,7 +1,9 @@ -from django.conf.urls import url +from django.urls import re_path from .views import ContentPermalinkRedirect urlpatterns = [ - url(r"^viewcontent$", ContentPermalinkRedirect.as_view(), name="contentpermalink"), + re_path( + r"^viewcontent$", ContentPermalinkRedirect.as_view(), name="contentpermalink" + ), ] diff --git a/kolibri/core/content/utils/channel_import.py b/kolibri/core/content/utils/channel_import.py index 1272c9a37ae..e744866d672 100644 --- a/kolibri/core/content/utils/channel_import.py +++ b/kolibri/core/content/utils/channel_import.py @@ -918,6 +918,11 @@ def import_channel_data(self): try: self.try_attaching_sqlite_database() transaction = self.destination.connection.begin() + if self.destination.engine.name == "sqlite": + # turn off foreign key integrity checking for the duration of the transaction + # so that we get similar behaviour to Postgresql, where the integrity of foreign + # keys is checked at the end of the transaction, rather than after each statement + self.destination.execute("PRAGMA foreign_keys=OFF") if self.check_and_delete_existing_channel(): for model in self.content_models: model_start = time.time() @@ -933,6 +938,9 @@ def import_channel_data(self): ) ) import_ran = True + if self.destination.engine.name == "sqlite": + # reenable foreign key integrity checking before we commit the transaction + self.destination.execute("PRAGMA foreign_keys=ON") transaction.commit() self.try_detaching_sqlite_database() diff --git a/kolibri/core/content/utils/content_request.py b/kolibri/core/content/utils/content_request.py index c3fde60dae9..a1dba9e803d 100644 --- a/kolibri/core/content/utils/content_request.py +++ b/kolibri/core/content/utils/content_request.py @@ -64,18 +64,6 @@ def _uuid_to_hex(_uuid): return _uuid.hex if isinstance(_uuid, uuid.UUID) else uuid.UUID(_uuid).hex -class FixedExists(Exists): - """ - Exists() subquery that allows positional arguments, to get around issue: - TypeError: resolve_expression() takes from 1 to 2 positional arguments but 6 were given - """ - - def resolve_expression(self, query=None, *args, **kwargs): - # @see Exists.resolve_expression - self.queryset = self.queryset.order_by() - return Subquery.resolve_expression(self, query, *args, **kwargs) - - def create_content_download_requests(facility, assignments, source_instance_id=None): """ Creates sync-initiated content download requests and removes corresponding removals @@ -425,7 +413,7 @@ def incomplete_downloads_queryset(): is_learner_download=Case( When( source_model=FacilityUser.morango_model_name, - then=FixedExists( + then=Exists( FacilityUser.objects.filter( id=OuterRef("source_id"), roles__isnull=True, diff --git a/kolibri/core/content/zip_wsgi.py b/kolibri/core/content/zip_wsgi.py index 7cf0da8145a..1411ac246f0 100644 --- a/kolibri/core/content/zip_wsgi.py +++ b/kolibri/core/content/zip_wsgi.py @@ -33,14 +33,14 @@ def add_security_headers(request, response): - response["Access-Control-Allow-Origin"] = "*" - response["Access-Control-Allow-Methods"] = "GET, OPTIONS" + response.headers["Access-Control-Allow-Origin"] = "*" + response.headers["Access-Control-Allow-Methods"] = "GET, OPTIONS" requested_headers = request.META.get("HTTP_ACCESS_CONTROL_REQUEST_HEADERS", "") if requested_headers: - response["Access-Control-Allow-Headers"] = requested_headers + response.headers["Access-Control-Allow-Headers"] = requested_headers # restrict CSP to only allow resources to be loaded from self, to prevent info leakage # (e.g. via passing user info out as GET parameters to an attacker's server), or inadvertent data usage - response[ + response.headers[ "Content-Security-Policy" ] = "default-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:" @@ -162,7 +162,7 @@ def get_embedded_file(zipped_path, zipped_filename, embedded_filepath): # set the content-length header to the size of the embedded file if file_size: - response["Content-Length"] = file_size + response.headers["Content-Length"] = file_size return response @@ -289,9 +289,9 @@ def _zip_content_from_request(request): # noqa: C901 raise # ensure the browser knows not to try byte-range requests, as we don't support them here - response["Accept-Ranges"] = "none" + response.headers["Accept-Ranges"] = "none" - response["Last-Modified"] = http_date(time.time()) + response.headers["Last-Modified"] = http_date(time.time()) patch_response_headers(response, cache_timeout=YEAR_IN_SECONDS) diff --git a/kolibri/core/decorators.py b/kolibri/core/decorators.py index 13768233b7f..7335c688018 100644 --- a/kolibri/core/decorators.py +++ b/kolibri/core/decorators.py @@ -368,7 +368,7 @@ def inner_func(*args, **kwargs): render_and_cache(response, CACHE_KEY_TEMPLATE.format(request.path)) patch_response_headers(response, cache_timeout=CACHE_TIMEOUT) - response["Vary"] = "accept-encoding, accept" + response.headers["Vary"] = "accept-encoding, accept" return response return inner_func diff --git a/kolibri/core/device/api.py b/kolibri/core/device/api.py index a966fcec096..7d6b1f68013 100644 --- a/kolibri/core/device/api.py +++ b/kolibri/core/device/api.py @@ -28,6 +28,7 @@ from rest_framework import viewsets from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response +from rest_framework.serializers import Serializer import kolibri from .models import DevicePermissions @@ -123,6 +124,18 @@ def create(self, request, *args, **kwargs): class FreeSpaceView(mixins.ListModelMixin, viewsets.GenericViewSet): permission_classes = (IsAuthenticated,) + def get_serializer_class(self): + """ + Add this purely to avoid warnings from DRF YASG schema generation. + """ + return Serializer + + def get_queryset(self): + """ + Add this purely to avoid warnings from DRF YASG schema generation. + """ + return None + def list(self, request): path = request.query_params.get("path") if path is None: @@ -297,7 +310,7 @@ class UserSyncStatusViewSet(ReadOnlyValuesViewset): permission_classes = (KolibriAuthPermissions,) filter_backends = (KolibriAuthPermissionsFilter, DjangoFilterBackend) queryset = UserSyncStatus.objects.all() - filter_class = SyncStatusFilter + filterset_class = SyncStatusFilter values = ( "status", diff --git a/kolibri/core/device/api_urls.py b/kolibri/core/device/api_urls.py index ac21eb67072..29a4b4ac689 100644 --- a/kolibri/core/device/api_urls.py +++ b/kolibri/core/device/api_urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from rest_framework import routers from .api import DeviceInfoView @@ -22,16 +22,16 @@ urlpatterns = [ - url(r"^", include(router.urls)), - url( + re_path(r"^", include(router.urls)), + re_path( r"^deviceprovision/", DeviceProvisionView.as_view({"post": "create"}), name="deviceprovision", ), - url(r"^freespace/", FreeSpaceView.as_view({"get": "list"}), name="freespace"), - url(r"^deviceinfo/", DeviceInfoView.as_view(), name="deviceinfo"), - url(r"^devicesettings/", DeviceSettingsView.as_view(), name="devicesettings"), - url(r"^devicename/", DeviceNameView.as_view(), name="devicename"), - url(r"^devicerestart/", DeviceRestartView.as_view(), name="devicerestart"), - url(r"^pathpermission/", PathPermissionView.as_view(), name="pathpermission"), + re_path(r"^freespace/", FreeSpaceView.as_view({"get": "list"}), name="freespace"), + re_path(r"^deviceinfo/", DeviceInfoView.as_view(), name="deviceinfo"), + re_path(r"^devicesettings/", DeviceSettingsView.as_view(), name="devicesettings"), + re_path(r"^devicename/", DeviceNameView.as_view(), name="devicename"), + re_path(r"^devicerestart/", DeviceRestartView.as_view(), name="devicerestart"), + re_path(r"^pathpermission/", PathPermissionView.as_view(), name="pathpermission"), ] diff --git a/kolibri/core/device/middleware.py b/kolibri/core/device/middleware.py index 29bb1c36837..3f04d48428e 100644 --- a/kolibri/core/device/middleware.py +++ b/kolibri/core/device/middleware.py @@ -15,7 +15,7 @@ class KolibriLocaleMiddleware(object): """ Copied and then modified into a new style middleware from: - https://github.com/django/django/blob/stable/1.11.x/django/middleware/locale.py + https://github.com/django/django/blob/stable/3.2.x/django/middleware/locale.py#L10 Also has several other changes to suit our purposes. The principal concern of this middleware is to activate translation for the current language, so that throughout the lifecycle of this request, any translation or language @@ -80,8 +80,8 @@ def __call__(self, request): return HttpResponseRedirect(language_url) # Add a content language header to the response if not already present. - if "Content-Language" not in response: - response["Content-Language"] = language + if "Content-Language" not in response.headers: + response.headers["Content-Language"] = language return response @@ -132,7 +132,7 @@ def process_exception(self, request, exception): "Database is not available for write operations", status=503, ) - response["Retry-After"] = 10 + response.headers["Retry-After"] = 10 return response def __call__(self, request): diff --git a/kolibri/core/device/migrations/0022_alter_usersyncstatus_user.py b/kolibri/core/device/migrations/0022_alter_usersyncstatus_user.py new file mode 100644 index 00000000000..5550f62b257 --- /dev/null +++ b/kolibri/core/device/migrations/0022_alter_usersyncstatus_user.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.23 on 2024-02-10 22:08 +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("kolibriauth", "0001_initial"), + ("device", "0021_default_demographic_fields"), + ] + + operations = [ + migrations.AlterField( + model_name="usersyncstatus", + name="user", + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + to="kolibriauth.FacilityUser", + ), + ), + ] diff --git a/kolibri/core/device/models.py b/kolibri/core/device/models.py index 5427455c24e..ae8f36de06a 100644 --- a/kolibri/core/device/models.py +++ b/kolibri/core/device/models.py @@ -6,6 +6,7 @@ from django.db import models from django.db.models import F from django.db.models import QuerySet +from django.db.utils import IntegrityError from morango.models import UUIDField from morango.models.core import InstanceIDModel from morango.models.core import SyncSession @@ -498,9 +499,7 @@ def allow_migrate(self, db, app_label, model_name=None, **hints): class UserSyncStatus(models.Model): - user = models.ForeignKey( - FacilityUser, on_delete=models.CASCADE, null=False, unique=True - ) + user = models.OneToOneField(FacilityUser, on_delete=models.CASCADE, null=False) # the last sync session sync_session = models.ForeignKey( SyncSession, on_delete=models.SET_NULL, null=True, blank=True @@ -563,8 +562,14 @@ def update_status(cls, user_id): # Only update the sync_session_id if it is not None, as otherwise we will be clearing # historical data that is used by the sync status API defaults["sync_session_id"] = sync_session_id - - cls.objects.update_or_create(user_id=user_id, defaults=defaults) + try: + cls.objects.update_or_create(user_id=user_id, defaults=defaults) + except IntegrityError: + # If we get an IntegrityError, it probably means that the user does not exist locally yet. + # This can happen if the user was created on the server and has not been synced down yet. + # In this case, we will just ignore the error as the sync status + # will be updated when the sync finalizes. + pass @property def queued(self): diff --git a/kolibri/core/device/serializers.py b/kolibri/core/device/serializers.py index 9ac0e4aea85..eebe98feb86 100644 --- a/kolibri/core/device/serializers.py +++ b/kolibri/core/device/serializers.py @@ -1,6 +1,6 @@ from django.db import transaction from django.utils.translation import check_for_language -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from rest_framework.exceptions import ParseError diff --git a/kolibri/core/device/soud.py b/kolibri/core/device/soud.py index 3b48107a533..e5f5cc8cee2 100644 --- a/kolibri/core/device/soud.py +++ b/kolibri/core/device/soud.py @@ -375,11 +375,18 @@ def execute_sync(context): sync_session_id = sync_queue.sync_session_id cleanup = False command = "sync" - resume_kwargs = {} + kwargs = dict( + user=context.user_id, + baseurl=context.network_location.base_url, + keep_alive=True, + noninteractive=True, + ) if sync_session_id: command = "resumesync" - resume_kwargs["id"] = sync_session_id + kwargs["id"] = sync_session_id + else: + kwargs["facility"] = context.user.facility_id sync_queue.status = SyncQueueStatus.Syncing sync_queue.save() @@ -389,15 +396,7 @@ def execute_sync(context): if not context.network_location: raise NetworkLocation.DoesNotExist - call_command( - command, - user=context.user_id, - facility=context.user.facility_id, - baseurl=context.network_location.base_url, - keep_alive=True, - noninteractive=True, - **resume_kwargs - ) + call_command(command, **kwargs) except NetworkLocation.DoesNotExist: cleanup = True logger.debug("{} Network location unavailable".format(context)) diff --git a/kolibri/core/device/task_notifications.py b/kolibri/core/device/task_notifications.py index 5042ae8f53b..98c0ea87f93 100644 --- a/kolibri/core/device/task_notifications.py +++ b/kolibri/core/device/task_notifications.py @@ -3,7 +3,7 @@ TODO: This can be migrated into kolibri/core/device/tasks.py in 0.17 """ from kolibri.core.tasks.job import JobStatus -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ def status_fn(job): diff --git a/kolibri/core/device/test/locale_middleware_urls.py b/kolibri/core/device/test/locale_middleware_urls.py index 7c01ee266dd..72c91698137 100644 --- a/kolibri/core/device/test/locale_middleware_urls.py +++ b/kolibri/core/device/test/locale_middleware_urls.py @@ -1,7 +1,8 @@ from __future__ import unicode_literals -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import path +from django.urls import re_path from django.views.generic import TemplateView from kolibri.core.device.translation import i18n_patterns @@ -10,18 +11,18 @@ view = TemplateView.as_view(template_name="dummy.html") -included = [url(r"^foo/$", view, name="not-prefixed-included-url")] +included = [re_path(r"^foo/$", view, name="not-prefixed-included-url")] patterns = [ - url(r"^not-prefixed/$", view, name="not-prefixed"), - url(r"^not-prefixed-include/", include(included)), + re_path(r"^not-prefixed/$", view, name="not-prefixed"), + re_path(r"^not-prefixed-include/", include(included)), ] patterns += i18n_patterns( [ - url(r"^prefixed/$", view, name="prefixed"), - url(r"^prefixed\.xml$", view, name="prefixed_xml"), + re_path(r"^prefixed/$", view, name="prefixed"), + re_path(r"^prefixed\.xml$", view, name="prefixed_xml"), ] ) -urlpatterns = [url(path_prefix, include(patterns))] +urlpatterns = [path(path_prefix, include(patterns))] diff --git a/kolibri/core/device/test/prefixed_locale_middleware_urls.py b/kolibri/core/device/test/prefixed_locale_middleware_urls.py index faeaa47bfb7..213d824b4d6 100644 --- a/kolibri/core/device/test/prefixed_locale_middleware_urls.py +++ b/kolibri/core/device/test/prefixed_locale_middleware_urls.py @@ -1,10 +1,10 @@ from __future__ import unicode_literals -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import path from .locale_middleware_urls import patterns path_prefix = "test/" -urlpatterns = [url(path_prefix, include(patterns))] +urlpatterns = [path(path_prefix, include(patterns))] diff --git a/kolibri/core/device/test/test_locale_middleware.py b/kolibri/core/device/test/test_locale_middleware.py index 16bb26d37a9..b5d42d5b0c0 100644 --- a/kolibri/core/device/test/test_locale_middleware.py +++ b/kolibri/core/device/test/test_locale_middleware.py @@ -8,7 +8,6 @@ from django.urls import clear_url_caches from django.urls import reverse from django.utils import translation -from django.utils._os import upath from mock import patch from kolibri.core.auth.test.helpers import clear_process_cache @@ -29,7 +28,7 @@ "TEMPLATES": [ { "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [os.path.join(os.path.dirname(upath(__file__)), "templates")], + "DIRS": [os.path.join(os.path.dirname(__file__), "templates")], } ], } diff --git a/kolibri/core/device/test/test_models.py b/kolibri/core/device/test/test_models.py index 39b9f423aaf..73b458d426b 100644 --- a/kolibri/core/device/test/test_models.py +++ b/kolibri/core/device/test/test_models.py @@ -18,7 +18,7 @@ class SyncQueueTestCase(TestCase): - multi_db = True + databases = "__all__" def setUp(self): clear_process_cache() diff --git a/kolibri/core/device/test/test_soud.py b/kolibri/core/device/test/test_soud.py index 9e3a1a3d43f..d3cca9730d1 100644 --- a/kolibri/core/device/test/test_soud.py +++ b/kolibri/core/device/test/test_soud.py @@ -26,6 +26,8 @@ class SoudContextTestCase(TestCase): + databases = "__all__" + def setUp(self): super(SoudContextTestCase, self).setUp() self.context = Context(uuid.uuid4().hex, uuid.uuid4().hex) @@ -59,7 +61,7 @@ def test_property__network_location__not_connected(self): class SoudRequestSyncHookHandlerTestCase(TestCase): - multi_db = True + databases = "__all__" def setUp(self): super(SoudRequestSyncHookHandlerTestCase, self).setUp() @@ -171,7 +173,7 @@ def test_multiple_users(self): @mute_signals(post_save) class SoudExecuteSyncsTestCase(TestCase): - multi_db = True + databases = "__all__" def setUp(self): super(SoudExecuteSyncsTestCase, self).setUp() @@ -373,7 +375,7 @@ def test_ordering(self): @mute_signals(post_save) class SoudExecuteSyncTestCase(TestCase): - multi_db = True + databases = "__all__" def setUp(self): super(SoudExecuteSyncTestCase, self).setUp() diff --git a/kolibri/core/device/translation.py b/kolibri/core/device/translation.py index 7abcf668287..a3811cd864d 100644 --- a/kolibri/core/device/translation.py +++ b/kolibri/core/device/translation.py @@ -8,7 +8,7 @@ from django.conf import settings from django.urls import resolve from django.urls import Resolver404 -from django.urls.resolvers import RegexURLResolver +from django.urls import URLResolver from django.utils.translation import get_language from django.utils.translation import LANGUAGE_SESSION_KEY from django.utils.translation.trans_real import check_for_language @@ -110,7 +110,7 @@ def get_language_from_request_and_is_from_path(request): # noqa complexity-16 def i18n_patterns(urls, prefix=None): """ Add the language code prefix to every URL pattern within this function. - Vendored from https://github.com/django/django/blob/stable/1.11.x/django/conf/urls/i18n.py + Vendored from https://github.com/django/django/blob/stable/3.2.x/django/conf/urls/i18n.py#L8 to allow use of this outside of the root URL conf to prefix plugin non-api urls. """ if not settings.USE_I18N: @@ -124,43 +124,53 @@ def recurse_urls_and_set(urls_to_set): setattr(url.callback, "translated", True) recurse_urls_and_set(urls) - return [LocaleRegexURLResolver(list(urls), prefix=prefix)] + return [ + URLResolver( + LocalePrefixPattern(prefix=prefix), + list(urls), + ) + ] -class LocaleRegexURLResolver(RegexURLResolver): +class LocalePrefixPattern: """ - A URL resolver that always matches the active language code as URL prefix. - Rather than taking a regex argument, we just override the ``regex`` - function to always return the active language-code as regex. - Vendored from https://github.com/django/django/blob/stable/1.11.x/django/urls/resolvers.py + A Locale prefix pattern that uses our device language setting for the active language. + It also allows passing a prefix to allow nested i18n_patterns to work correctly. + Vendored from https://github.com/django/django/blob/stable/3.2.x/django/urls/resolvers.py#L298 As using the Django internal version inside included URL configs is disallowed. Rather than monkey patch Django to allow this for our use case, make a copy of this here and use this instead. """ - def __init__( - self, - urlconf_name, - default_kwargs=None, - app_name=None, - namespace=None, - prefix_default_language=True, - prefix=None, - ): - super(LocaleRegexURLResolver, self).__init__( - None, urlconf_name, default_kwargs, app_name, namespace - ) + def __init__(self, prefix=None, prefix_default_language=True): self.prefix_default_language = prefix_default_language + self.converters = {} self._prefix = prefix @property def regex(self): + # This is only used by reverse() and cached in _reverse_dict. + return re.compile(re.escape(self.language_prefix)) + + @property + def language_prefix(self): device_language = get_device_language() or get_settings_language() language_code = get_language() or device_language - if language_code not in self._regex_dict: - if language_code == device_language and not self.prefix_default_language: - regex_string = self._prefix or "" - else: - regex_string = ("^%s/" % language_code) + (self._prefix or "") - self._regex_dict[language_code] = re.compile(regex_string, re.UNICODE) - return self._regex_dict[language_code] + if language_code == device_language and not self.prefix_default_language: + return self._prefix or "" + return ("%s/" % language_code) + (self._prefix or "") + + def match(self, path): + language_prefix = self.language_prefix + if path.startswith(language_prefix): + return path[len(language_prefix) :], (), {} + return None + + def check(self): + return [] + + def describe(self): + return "'{}'".format(self) + + def __str__(self): + return self.language_prefix diff --git a/kolibri/core/deviceadmin/tests/test_dbbackup.py b/kolibri/core/deviceadmin/tests/test_dbbackup.py index ddee7b5d220..e62d450c200 100644 --- a/kolibri/core/deviceadmin/tests/test_dbbackup.py +++ b/kolibri/core/deviceadmin/tests/test_dbbackup.py @@ -25,6 +25,7 @@ def test_active_kolibri(): gs.assert_called_once() +@pytest.mark.django_db def test_inactive_kolibri(): """ Tests that if kolibri is inactive, a dump is created diff --git a/kolibri/core/deviceadmin/tests/test_dbrestore.py b/kolibri/core/deviceadmin/tests/test_dbrestore.py index ed6351fc66f..92eb712f30e 100644 --- a/kolibri/core/deviceadmin/tests/test_dbrestore.py +++ b/kolibri/core/deviceadmin/tests/test_dbrestore.py @@ -59,13 +59,13 @@ def test_latest(): def test_illegal_command(): - with pytest.raises(CommandError): + with pytest.raises(ValueError): call_command("dbrestore", latest=True, dump_file="wup wup") def test_no_restore_from_no_file(): - with pytest.raises(CommandError): + with pytest.raises(ValueError): call_command("dbrestore", dump_file="does not exist") diff --git a/kolibri/core/deviceadmin/utils.py b/kolibri/core/deviceadmin/utils.py index 4c5e6b2bfc0..9c3bc6940a5 100644 --- a/kolibri/core/deviceadmin/utils.py +++ b/kolibri/core/deviceadmin/utils.py @@ -142,7 +142,9 @@ def dbrestore(from_file): # See: https://github.com/learningequality/kolibri/issues/2875 with open(from_file, **KWARGS_IO_READ) as f: db.connections["default"].connect() + db.connections["default"].connection.execute("PRAGMA foreign_keys=OFF") db.connections["default"].connection.executescript(f.read()) + db.connections["default"].connection.execute("PRAGMA foreign_keys=ON") # Finally, it's okay to import models and open database connections. # We need this to avoid generating records with identical 'Instance ID' diff --git a/kolibri/core/discovery/api.py b/kolibri/core/discovery/api.py index 8b06a3752c9..0e1b7943cae 100644 --- a/kolibri/core/discovery/api.py +++ b/kolibri/core/discovery/api.py @@ -17,6 +17,7 @@ from .utils.network.client import NetworkClient from .utils.network.connections import capture_connection_state from .utils.network.connections import update_network_location +from kolibri.core.api import BaseValuesViewset from kolibri.core.api import ValuesViewset from kolibri.core.device.permissions import NotProvisionedHasPermission from kolibri.core.utils.urls import reverse_path @@ -74,7 +75,8 @@ class StaticNetworkLocationViewSet(NetworkLocationViewSet): queryset = StaticNetworkLocation.objects.all() -class NetworkLocationFacilitiesView(viewsets.GenericViewSet): +class NetworkLocationFacilitiesView(BaseValuesViewset): + queryset = NetworkLocation.objects.all() permission_classes = [NetworkLocationPermissions | NotProvisionedHasPermission] def retrieve(self, request, pk=None): diff --git a/kolibri/core/discovery/apps.py b/kolibri/core/discovery/apps.py deleted file mode 100644 index aa13278fb5c..00000000000 --- a/kolibri/core/discovery/apps.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import unicode_literals - -from django.apps import AppConfig - - -class DiscoveryConfig(AppConfig): - name = "discovery" diff --git a/kolibri/core/discovery/models.py b/kolibri/core/discovery/models.py index 9ecaa050040..ed2b776ae44 100644 --- a/kolibri/core/discovery/models.py +++ b/kolibri/core/discovery/models.py @@ -1,5 +1,6 @@ import uuid +from django.core.exceptions import FieldDoesNotExist from django.core.exceptions import ValidationError from django.db import models from django.utils import timezone @@ -154,7 +155,7 @@ def has_field(cls, field): try: cls._meta.get_field(field) return True - except models.FieldDoesNotExist: + except FieldDoesNotExist: return False def matches_version(self, version): diff --git a/kolibri/core/discovery/test/test_api.py b/kolibri/core/discovery/test/test_api.py index 97367c5591e..2ceb684960b 100644 --- a/kolibri/core/discovery/test/test_api.py +++ b/kolibri/core/discovery/test/test_api.py @@ -23,6 +23,8 @@ @mock.patch.object(requests.Session, "request", mock_request) @mock.patch.object(connections, "check_if_port_open", lambda *a: True) class NetworkLocationAPITestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): provision_device() @@ -132,6 +134,8 @@ def test_reading_network_location_list_filter_soud(self): class PinnedDeviceAPITestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): provision_device() diff --git a/kolibri/core/discovery/test/test_connections.py b/kolibri/core/discovery/test/test_connections.py index 83e8fe5b40d..a78f1f4107b 100644 --- a/kolibri/core/discovery/test/test_connections.py +++ b/kolibri/core/discovery/test/test_connections.py @@ -13,6 +13,8 @@ class BaseTestCase(TestCase): + databases = "__all__" + def setUp(self): self.mock_location = mock.MagicMock( spec=NetworkLocation(), diff --git a/kolibri/core/discovery/test/test_models.py b/kolibri/core/discovery/test/test_models.py index 98b61cbba72..0d976c19741 100644 --- a/kolibri/core/discovery/test/test_models.py +++ b/kolibri/core/discovery/test/test_models.py @@ -8,7 +8,7 @@ class NetworkLocationTestCase(TestCase): - multi_db = True + databases = "__all__" def test_property__available(self): location = NetworkLocation() diff --git a/kolibri/core/discovery/test/test_network_search.py b/kolibri/core/discovery/test/test_network_search.py index d1180cfb887..bbfa348d7ab 100644 --- a/kolibri/core/discovery/test/test_network_search.py +++ b/kolibri/core/discovery/test/test_network_search.py @@ -17,7 +17,7 @@ class NetworkLocationListenerTestCase(TransactionTestCase): - multi_db = True + databases = "__all__" def setUp(self): super(NetworkLocationListenerTestCase, self).setUp() diff --git a/kolibri/core/discovery/test/test_tasks.py b/kolibri/core/discovery/test/test_tasks.py index 387aedc6714..6b5c7570b25 100644 --- a/kolibri/core/discovery/test/test_tasks.py +++ b/kolibri/core/discovery/test/test_tasks.py @@ -37,7 +37,7 @@ def unwrap(func): class PerformNetworkLocationUpdateTestCase(TestCase): - multi_db = True + databases = "__all__" def setUp(self): self.network_location = DynamicNetworkLocation.objects.create( @@ -106,7 +106,7 @@ def test_skip_enqueue_another(self, mock_update, mock_enqueue_another): class AddDynamicNetworkLocationTestCase(TestCase): - multi_db = True + databases = "__all__" def setUp(self): self.broadcast_id = uuid.uuid4().hex @@ -162,7 +162,7 @@ def test_added__both_souds(self, mock_enqueue_update, mock_get_device_setting): class RemoveDynamicNetworkLocationTestCase(TestCase): - multi_db = True + databases = "__all__" def setUp(self): self.broadcast_id = uuid.uuid4().hex @@ -212,7 +212,7 @@ def test_dispatch(self, mock_dispatch): class DispatchBroadcastHooksTestCase(TestCase): - multi_db = True + databases = "__all__" def setUp(self): self.broadcast_id = uuid.uuid4().hex @@ -271,7 +271,7 @@ def test_okay__no_method(self): class ResetConnectionStatesTestCase(TestCase): - multi_db = True + databases = "__all__" def setUp(self): self.old_broadcast_id = uuid.uuid4().hex @@ -356,7 +356,7 @@ def test_dynamic_already_added_for_new_broadcast( class TaskUtilitiesTestCase(TestCase): - multi_db = True + databases = "__all__" def setUp(self): self.broadcast_id = uuid.uuid4().hex diff --git a/kolibri/core/discovery/test/test_upgrade.py b/kolibri/core/discovery/test/test_upgrade.py index d5520300a98..65b5907686b 100644 --- a/kolibri/core/discovery/test/test_upgrade.py +++ b/kolibri/core/discovery/test/test_upgrade.py @@ -12,7 +12,7 @@ class TestNetworkLocationUpgrade(TestCase): - multi_db = True + databases = "__all__" @unittest.skipIf( getattr(settings, "DATABASES")["default"]["ENGINE"] diff --git a/kolibri/core/exams/api.py b/kolibri/core/exams/api.py index 10a7dee10ad..7e82d0be689 100644 --- a/kolibri/core/exams/api.py +++ b/kolibri/core/exams/api.py @@ -60,7 +60,7 @@ class ExamViewset(ValuesViewset): pagination_class = OptionalPageNumberPagination permission_classes = (ExamPermissions,) filter_backends = (KolibriAuthPermissionsFilter, DjangoFilterBackend) - filter_class = ExamFilter + filterset_class = ExamFilter values = ( "id", diff --git a/kolibri/core/exams/api_urls.py b/kolibri/core/exams/api_urls.py index e3f9d2c2dbe..9bf7cf8d4a3 100644 --- a/kolibri/core/exams/api_urls.py +++ b/kolibri/core/exams/api_urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from rest_framework import routers from .api import ExamViewset @@ -7,4 +7,4 @@ router = routers.SimpleRouter() router.register(r"exam", ExamViewset, basename="exam") -urlpatterns = [url(r"^", include(router.urls))] +urlpatterns = [re_path(r"^", include(router.urls))] diff --git a/kolibri/core/exams/single_user_assignment_utils.py b/kolibri/core/exams/single_user_assignment_utils.py index 314d5456563..95b822ea7ad 100644 --- a/kolibri/core/exams/single_user_assignment_utils.py +++ b/kolibri/core/exams/single_user_assignment_utils.py @@ -58,13 +58,17 @@ def update_assignments_from_individual_syncable_exams(user_id): syncableexams = IndividualSyncableExam.objects.filter(user_id=user_id) assignments = ExamAssignment.objects.filter( collection__membership__user_id=user_id, exam__active=True - ).distinct() + ) # get a list of all syncable exams that aren't locally assigned - to_create = syncableexams.exclude(exam_id__in=assignments.values_list("exam_id")) + to_create = syncableexams.exclude( + exam_id__in=assignments.values_list("exam_id") + ).distinct() # get a list of all assignments that may need updating from syncable exams - to_update = assignments.filter(exam_id__in=syncableexams.values_list("exam_id")) + to_update = assignments.filter( + exam_id__in=syncableexams.values_list("exam_id") + ).distinct() # get a list of all active assignments that no longer have a syncable exam to_delete = assignments.exclude(exam_id__in=syncableexams.values_list("exam_id")) diff --git a/kolibri/core/exams/test/test_exam_api.py b/kolibri/core/exams/test/test_exam_api.py index e6db25e9fd1..30594ac4287 100644 --- a/kolibri/core/exams/test/test_exam_api.py +++ b/kolibri/core/exams/test/test_exam_api.py @@ -21,6 +21,8 @@ class ExamAPITestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): provision_device() diff --git a/kolibri/core/fields.py b/kolibri/core/fields.py index 3abfce1e072..91d0f8b6c70 100644 --- a/kolibri/core/fields.py +++ b/kolibri/core/fields.py @@ -63,10 +63,12 @@ class DateTimeTzField(Field): against this in the database. Mostly engineered for SQLite usage. """ + morango_serialize_to_string = True + def db_type(self, connection): return "varchar" - def from_db_value(self, value, expression, connection, context): + def from_db_value(self, value, expression, connection): if value is None: return value return parse_timezonestamp(value) @@ -93,13 +95,16 @@ def get_db_prep_value(self, value, connection, prepared=False): value = self.get_prep_value(value) return value - def value_from_object_json_compatible(self, obj): - if self.value_from_object(obj): + def value_to_string(self, obj): + value = self.value_from_object(obj) + if value is not None: return create_timezonestamp(self.value_from_object(obj)) class JSONField(JSONFieldBase): - def from_db_value(self, value, expression, connection, context): + morango_serialize_to_string = True + + def from_db_value(self, value, expression, connection): if isinstance(value, str): try: return json.loads(value, **self.load_kwargs) diff --git a/kolibri/core/lessons/api_urls.py b/kolibri/core/lessons/api_urls.py index 4cf04488ddf..82d1ece1226 100644 --- a/kolibri/core/lessons/api_urls.py +++ b/kolibri/core/lessons/api_urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from rest_framework import routers from .viewsets import LessonViewset @@ -7,4 +7,4 @@ router = routers.SimpleRouter() router.register(r"lesson", LessonViewset, basename="lesson") -urlpatterns = [url(r"^", include(router.urls))] +urlpatterns = [re_path(r"^", include(router.urls))] diff --git a/kolibri/core/lessons/single_user_assignment_utils.py b/kolibri/core/lessons/single_user_assignment_utils.py index e3e2fbe0715..d0f9acb4ccc 100644 --- a/kolibri/core/lessons/single_user_assignment_utils.py +++ b/kolibri/core/lessons/single_user_assignment_utils.py @@ -68,17 +68,17 @@ def update_assignments_from_individual_syncable_lessons(user_id): syncablelessons = IndividualSyncableLesson.objects.filter(user_id=user_id) assignments = LessonAssignment.objects.filter( collection__membership__user_id=user_id, lesson__is_active=True - ).distinct() + ) # get a list of all syncable lessons that aren't locally assigned to_create = syncablelessons.exclude( lesson_id__in=assignments.values_list("lesson_id") - ) + ).distinct() # get a list of all assignments that may need updating from syncable lessons to_update = assignments.filter( lesson_id__in=syncablelessons.values_list("lesson_id") - ) + ).distinct() # get a list of all active assignments that no longer have a syncable lesson to_delete = assignments.exclude( diff --git a/kolibri/core/lessons/test/test_lesson_api.py b/kolibri/core/lessons/test/test_lesson_api.py index 5d701eb4800..57eb0c1834d 100644 --- a/kolibri/core/lessons/test/test_lesson_api.py +++ b/kolibri/core/lessons/test/test_lesson_api.py @@ -20,6 +20,8 @@ class LessonAPITestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): provision_device() diff --git a/kolibri/core/logger/api.py b/kolibri/core/logger/api.py index 402c3dd982d..948983256d9 100644 --- a/kolibri/core/logger/api.py +++ b/kolibri/core/logger/api.py @@ -31,6 +31,7 @@ from rest_framework.decorators import action from rest_framework.exceptions import ValidationError from rest_framework.response import Response +from rest_framework.serializers import Serializer from .models import AttemptLog from .models import ContentSessionLog @@ -230,6 +231,18 @@ def to_dict(self): @method_decorator(csrf_protect, name="dispatch") class ProgressTrackingViewSet(viewsets.GenericViewSet): + def get_serializer_class(self): + """ + Add this purely to avoid warnings from DRF YASG schema generation. + """ + return Serializer + + def get_queryset(self): + """ + Add this purely to avoid warnings from DRF YASG schema generation. + """ + return None + def _precache_dataset_id(self, user): if user is None or user.is_anonymous: return @@ -919,11 +932,21 @@ def update(self, request, pk=None): class TotalContentProgressViewSet(viewsets.GenericViewSet): + def get_serializer_class(self): + """ + Add this purely to avoid warnings from DRF YASG schema generation. + """ + return Serializer + + def get_queryset(self): + return ContentSummaryLog.objects.filter(user=self.request.user) + def retrieve(self, request, pk=None): if request.user.is_anonymous or pk != request.user.id: raise PermissionDenied("Can only access progress data for self") progress = ( - request.user.contentsummarylog_set.annotate( + self.get_queryset() + .annotate( mastery_progress=Sum( Case( When(masterylogs__complete=True, then=Value(1)), @@ -1013,7 +1036,7 @@ class MasteryLogViewSet(ReadOnlyValuesViewset): ) queryset = MasteryLog.objects.all().order_by(LOG_ORDER_BY) pagination_class = OptionalPageNumberPagination - filter_class = MasteryFilter + filterset_class = MasteryFilter values = ( "id", "mastery_criterion", @@ -1027,7 +1050,9 @@ class MasteryLogViewSet(ReadOnlyValuesViewset): def annotate_queryset(self, queryset): return queryset.annotate( - correct=Coalesce(Sum("attemptlogs__correct"), Value(0)) + correct=Coalesce( + Sum("attemptlogs__correct"), Value(0), output_field=IntegerField() + ) ) @action(detail=True) @@ -1101,7 +1126,7 @@ class AttemptLogViewSet(ReadOnlyValuesViewset): ) queryset = AttemptLog.objects.all() pagination_class = OptionalPageNumberPagination - filter_class = AttemptFilter + filterset_class = AttemptFilter values = attemptlog_values @@ -1136,7 +1161,7 @@ class GenerateCSVLogRequestViewSet(viewsets.ModelViewSet): filter_backends = (KolibriAuthPermissionsFilter, DjangoFilterBackend) queryset = GenerateCSVLogRequest.objects.all() serializer_class = GenerateCSVLogRequestSerializer - filter_class = GenerateCSVLogRequestFilter + filterset_class = GenerateCSVLogRequestFilter def _get_or_create_logrequest( self, diff --git a/kolibri/core/logger/test/test_api.py b/kolibri/core/logger/test/test_api.py index f91932926c2..e8c8ddc5bb7 100644 --- a/kolibri/core/logger/test/test_api.py +++ b/kolibri/core/logger/test/test_api.py @@ -39,6 +39,8 @@ class ContentSummaryLogCSVExportTestCase(APITestCase): + databases = "__all__" + fixtures = ["content_test.json"] @classmethod @@ -200,6 +202,7 @@ def test_csv_cleanup(self, mock_enqueue): class ContentSessionLogCSVExportTestCase(APITestCase): + databases = "__all__" fixtures = ["content_test.json"] diff --git a/kolibri/core/logger/utils/exam_log_migration.py b/kolibri/core/logger/utils/exam_log_migration.py index 949b292b474..328c6795227 100644 --- a/kolibri/core/logger/utils/exam_log_migration.py +++ b/kolibri/core/logger/utils/exam_log_migration.py @@ -1,9 +1,9 @@ from itertools import compress from itertools import groupby +from django.core.exceptions import FieldDoesNotExist from django.db import connections from django.db.models import F -from django.db.models import FieldDoesNotExist from django.db.models import Value from django.db.models.functions import Greatest from le_utils.constants import content_kinds @@ -139,7 +139,7 @@ def _create_attemptlog(examattemptlog, sessionlog_id, masterylog_id): try: field_obj = AttemptLog._meta.get_field(field) if hasattr(field_obj, "from_db_value"): - value = field_obj.from_db_value(value, None, None, None) + value = field_obj.from_db_value(value, None, None) except FieldDoesNotExist: pass setattr(attemptlog, field, value) diff --git a/kolibri/core/logger/utils/user_data.py b/kolibri/core/logger/utils/user_data.py index 66dd5ba5ce3..cd237d918e4 100644 --- a/kolibri/core/logger/utils/user_data.py +++ b/kolibri/core/logger/utils/user_data.py @@ -42,7 +42,7 @@ def logger_info(message, verbosity=1): # doesn't work on Windows, see: https://github.com/learningequality/kolibri/issues/7077 try: # MUST: Follow the verbosity mechanism of Django's management commands - # https://docs.djangoproject.com/en/1.11/ref/django-admin/#cmdoption-verbosity + # https://docs.djangoproject.com/en/3.2/ref/django-admin/#cmdoption-verbosity # and only show when it's > 0. # print("====> verbosity %s" % verbosity) if verbosity > 0: diff --git a/kolibri/core/notifications/models.py b/kolibri/core/notifications/models.py index cc929b5062b..26665e7801d 100644 --- a/kolibri/core/notifications/models.py +++ b/kolibri/core/notifications/models.py @@ -11,7 +11,6 @@ from django.conf import settings from django.db import models -from django.utils.encoding import python_2_unicode_compatible from morango.models import UUIDField from kolibri.core.fields import DateTimeTzField @@ -94,7 +93,6 @@ class HelpReason(ChoicesEnum): Multiple = "MultipleUnsuccessfulAttempts" -@python_2_unicode_compatible class LearnerProgressNotification(models.Model): id = ( models.AutoField( @@ -127,7 +125,6 @@ class Meta: app_label = "notifications" -@python_2_unicode_compatible class NotificationsLog(models.Model): id = ( models.AutoField( diff --git a/kolibri/core/notifications/test/test_api.py b/kolibri/core/notifications/test/test_api.py index 388fe4cd462..085331254db 100644 --- a/kolibri/core/notifications/test/test_api.py +++ b/kolibri/core/notifications/test/test_api.py @@ -1,4 +1,3 @@ -import json import uuid from datetime import timedelta @@ -49,6 +48,8 @@ class NotificationsAPITestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): provision_device() @@ -97,20 +98,18 @@ def setUp(self): is_active=True, created_by=self.superuser, collection=self.classroom, - resources=json.dumps( - [ - { - "contentnode_id": self.node_1.id, - "content_id": self.node_1.content_id, - "channel_id": self.channel_id, - }, - { - "contentnode_id": self.node_2.id, - "content_id": self.node_2.content_id, - "channel_id": self.channel_id, - }, - ] - ), + resources=[ + { + "contentnode_id": self.node_1.id, + "content_id": self.node_1.content_id, + "channel_id": self.channel_id, + }, + { + "contentnode_id": self.node_2.id, + "content_id": self.node_2.content_id, + "channel_id": self.channel_id, + }, + ], ) self.assignment_1 = LessonAssignment.objects.create( @@ -1124,6 +1123,8 @@ def test_parse_attemptslog_update_attempt_with_three_wrong_attempts_no_started( class BulkNotificationsAPITestCase(APITestCase): + databases = "__all__" + @classmethod def setUpTestData(cls): provision_device() @@ -1170,20 +1171,18 @@ def setUpTestData(cls): is_active=True, created_by=cls.superuser, collection=cls.classroom, - resources=json.dumps( - [ - { - "contentnode_id": cls.node_1.id, - "content_id": cls.node_1.content_id, - "channel_id": cls.channel_id, - }, - { - "contentnode_id": cls.node_2.id, - "content_id": cls.node_2.content_id, - "channel_id": cls.channel_id, - }, - ] - ), + resources=[ + { + "contentnode_id": cls.node_1.id, + "content_id": cls.node_1.content_id, + "channel_id": cls.channel_id, + }, + { + "contentnode_id": cls.node_2.id, + "content_id": cls.node_2.content_id, + "channel_id": cls.channel_id, + }, + ], ) cls.lesson_assignment = LessonAssignment.objects.create( diff --git a/kolibri/core/public/api.py b/kolibri/core/public/api.py index 8886b0130ed..7e7fccaca38 100644 --- a/kolibri/core/public/api.py +++ b/kolibri/core/public/api.py @@ -119,7 +119,7 @@ def wrapped_view(*args, **kwargs): patch_cache_control( response, max_age=300, stale_while_revalidate=100, public=True ) - response["Expires"] = http_date(time.time() + 300) + response.headers["Expires"] = http_date(time.time() + 300) return response return session_exempt(wrapped_view) diff --git a/kolibri/core/public/api_urls.py b/kolibri/core/public/api_urls.py index 2b56003a0ce..a165d1d9e3c 100644 --- a/kolibri/core/public/api_urls.py +++ b/kolibri/core/public/api_urls.py @@ -11,8 +11,8 @@ endpoint in place and maintained to the best extent possible so older clients can still use it. """ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from rest_framework import routers from ..auth.api import PublicFacilityUserViewSet @@ -58,24 +58,24 @@ # Add public api endpoints urlpatterns = [ - url(r"^", include(router.urls)), - url(r"v2/", include(public_content_v2_router.urls)), - url( - r"(?P[^/]+)/channels/lookup/(?P[^/]+)", + re_path(r"^", include(router.urls)), + re_path(r"v2/", include(public_content_v2_router.urls)), + re_path( + r"(?P[^/]+)/channels/lookup/(?P[^/]+)$", get_public_channel_lookup, name="get_public_channel_lookup", ), - url( + re_path( r"(?P[^/]+)/channels", get_public_channel_list, name="get_public_channel_list", ), - url( + re_path( r"(?P[^/]+)/file_checksums/", get_public_file_checksums, name="get_public_file_checksums", ), - url( + re_path( r"syncqueue/", SyncQueueAPIView.as_view(), name="syncqueue", diff --git a/kolibri/core/public/test/test_api.py b/kolibri/core/public/test/test_api.py index 2895617dfc1..4b4ecdf4506 100644 --- a/kolibri/core/public/test/test_api.py +++ b/kolibri/core/public/test/test_api.py @@ -283,7 +283,7 @@ class SyncQueueViewSetTestCase(APITestCase): to be changed, and not the tests themselves. """ - multi_db = True + databases = "__all__" def setUp(self): setup_device() diff --git a/kolibri/core/query.py b/kolibri/core/query.py index 020412e5972..529bf0e71e0 100644 --- a/kolibri/core/query.py +++ b/kolibri/core/query.py @@ -12,7 +12,7 @@ def __init__(self, *args, **kwargs): self.result_field = kwargs.pop("result_field", None) super(NotNullArrayAgg, self).__init__(*args, **kwargs) - def convert_value(self, value, expression, connection, context): + def convert_value(self, value, expression, connection): if not value: return [] results = list(filter(lambda x: x is not None, value)) @@ -56,7 +56,7 @@ def __init__(self, *args, **kwargs): self.result_field = kwargs.pop("result_field", None) super(GroupConcat, self).__init__(*args, **kwargs) - def convert_value(self, value, expression, connection, context): + def convert_value(self, value, expression, connection): if not value: return [] results = value.split(",") diff --git a/kolibri/core/serializers.py b/kolibri/core/serializers.py index e93694e4c52..51472e69a91 100644 --- a/kolibri/core/serializers.py +++ b/kolibri/core/serializers.py @@ -1,5 +1,5 @@ -from collections import Mapping from collections import OrderedDict +from collections.abc import Mapping import pytz from django.core.exceptions import ValidationError as DjangoValidationError diff --git a/kolibri/core/tasks/api.py b/kolibri/core/tasks/api.py index b400b21d40a..9f59464f240 100644 --- a/kolibri/core/tasks/api.py +++ b/kolibri/core/tasks/api.py @@ -46,6 +46,12 @@ def get_fields(self): class TasksViewSet(viewsets.GenericViewSet): serializer_class = TasksSerializer + def get_queryset(self): + """ + Add this purely to avoid warnings from DRF YASG schema generation. + """ + return None + def validate_create_req_data(self, request): """ Validates the request data received on POST /api/tasks/. diff --git a/kolibri/core/tasks/job.py b/kolibri/core/tasks/job.py index 31cc905e4b0..6c32e46f0cb 100644 --- a/kolibri/core/tasks/job.py +++ b/kolibri/core/tasks/job.py @@ -18,7 +18,7 @@ from kolibri.core.tasks.validation import validate_repeat from kolibri.core.tasks.validation import validate_timedelay from kolibri.utils import translation -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ logger = logging.getLogger(__name__) diff --git a/kolibri/core/templates/kolibri/base.html b/kolibri/core/templates/kolibri/base.html index 76226ba5104..fdd5191b693 100644 --- a/kolibri/core/templates/kolibri/base.html +++ b/kolibri/core/templates/kolibri/base.html @@ -1,5 +1,5 @@ {% load i18n core_tags webpack_tags content_tags cache %} -{% load staticfiles %} +{% load static %} {% get_current_language_bidi as LANGUAGE_BIDI %} {% get_current_language as LANGUAGE_CODE %} diff --git a/kolibri/core/templates/kolibri/unsupported_browser.html b/kolibri/core/templates/kolibri/unsupported_browser.html index 88e3cb0483f..0349427bb4e 100644 --- a/kolibri/core/templates/kolibri/unsupported_browser.html +++ b/kolibri/core/templates/kolibri/unsupported_browser.html @@ -1,5 +1,5 @@ {% load i18n core_tags %} -{% load staticfiles %} +{% load static %} {% get_current_language_bidi as LANGUAGE_BIDI %} {% get_current_language as LANGUAGE_CODE %} diff --git a/kolibri/core/templatetags/core_tags.py b/kolibri/core/templatetags/core_tags.py index 45f74d2599b..dc385bd08e3 100644 --- a/kolibri/core/templatetags/core_tags.py +++ b/kolibri/core/templatetags/core_tags.py @@ -10,7 +10,7 @@ from kolibri.core.hooks import FrontEndBaseHeadHook from kolibri.core.hooks import FrontEndBaseSyncHook from kolibri.core.theme_hook import ThemeHook -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ register = template.Library() diff --git a/kolibri/core/test/test_key_urls.py b/kolibri/core/test/test_key_urls.py index 0b8a558215f..24c5422d41e 100644 --- a/kolibri/core/test/test_key_urls.py +++ b/kolibri/core/test/test_key_urls.py @@ -74,6 +74,8 @@ def test_class_coach_is_redirected_to_coach_plugin(self): class AllUrlsTest(APITransactionTestCase): + databases = "__all__" + # Allow codes that may indicate a poorly formed response # 412 is returned from endpoints that have required GET params when these are not supplied allowed_http_codes = [200, 301, 302, 400, 401, 403, 404, 405, 412] @@ -100,7 +102,7 @@ def check_responses(self, credentials=None): # noqa max-complexity=12 the client while testing, and login again Specify @default_kwargs to be used for patterns that expect keyword parameters, e.g. if you specify default_kwargs={'username': 'testuser'}, then - for pattern url(r'^accounts/(?P[\.\w-]+)/$' + for pattern re_path(r'^accounts/(?P[\.\w-]+)/$' the url /accounts/testuser/ will be tested. If @quiet=False, print all the urls checked. If status code of the response is not 200, print the status code. @@ -122,7 +124,7 @@ def check_urls(urlpatterns, prefix=""): ) check_urls(pattern.url_patterns, prefix=new_prefix) skip = False - regex = pattern.regex + regex = pattern.pattern.regex if regex.groups > 0: skip = True if hasattr(pattern, "name") and pattern.name: diff --git a/kolibri/core/test/test_setlanguage.py b/kolibri/core/test/test_setlanguage.py index dc9e8a440ab..10a63c99dd4 100644 --- a/kolibri/core/test/test_setlanguage.py +++ b/kolibri/core/test/test_setlanguage.py @@ -81,12 +81,13 @@ def test_setlang_null(self): translate_url(reverse("kolibri:core:redirect_user"), lang_code), ) self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code) - post_data = dict(language=None) + post_data = {} response = self.client.post(reverse("kolibri:core:set_language"), post_data) + current_language = get_language() self.assertEqual(response.status_code, 200) self.assertEqual( response.content.decode("utf-8"), - translate_url(reverse("kolibri:core:redirect_user"), "en"), + translate_url(reverse("kolibri:core:redirect_user"), current_language), ) self.assertNotIn(LANGUAGE_SESSION_KEY, self.client.session) @@ -105,12 +106,15 @@ def test_setlang_null_next_valid(self): ) self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code) next_url = reverse("kolibri:kolibri.plugins.learn:learn") - post_data = dict(language=None, next=next_url) + post_data = dict(next=next_url) + current_language = get_language() response = self.client.post(reverse("kolibri:core:set_language"), post_data) self.assertEqual(response.status_code, 200) self.assertEqual( response.content.decode("utf-8"), - translate_url(reverse("kolibri:kolibri.plugins.learn:learn"), "en"), + translate_url( + reverse("kolibri:kolibri.plugins.learn:learn"), current_language + ), ) self.assertNotIn(LANGUAGE_SESSION_KEY, self.client.session) @@ -129,12 +133,13 @@ def test_setlang_null_next_invalid(self): ) self.assertEqual(self.client.session[LANGUAGE_SESSION_KEY], lang_code) next_url = "/not/a/real/url" - post_data = dict(language=None, next=next_url) + post_data = dict(next=next_url) response = self.client.post(reverse("kolibri:core:set_language"), post_data) + current_language = get_language() self.assertEqual(response.status_code, 200) self.assertEqual( response.content.decode("utf-8"), - translate_url(reverse("kolibri:core:redirect_user"), "en"), + translate_url(reverse("kolibri:core:redirect_user"), current_language), ) self.assertNotIn(LANGUAGE_SESSION_KEY, self.client.session) diff --git a/kolibri/core/test/test_utils.py b/kolibri/core/test/test_utils.py index 986afb20417..3d10644954d 100644 --- a/kolibri/core/test/test_utils.py +++ b/kolibri/core/test/test_utils.py @@ -15,7 +15,7 @@ class DBBasedProcessLockTestCase(SimpleTestCase): - allow_database_queries = True + databases = "__all__" @unittest.skipIf( True, diff --git a/kolibri/core/urls.py b/kolibri/core/urls.py index 7c2ce762f37..12a91a5dc85 100644 --- a/kolibri/core/urls.py +++ b/kolibri/core/urls.py @@ -30,8 +30,8 @@ Place a url.py and have your plugin's definition class's ``url_module`` method return the module. """ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from rest_framework import routers from .views import GuestRedirectView @@ -52,26 +52,26 @@ # Patterns that we want to prefix because they need access to the current language lang_prefixed_patterns = [ - url(r"^i18n/setlang/$", set_language, name="set_language"), - url(r"^logout/$", logout_view, name="logout"), - url(r"^redirectuser/$", RootURLRedirectView.as_view(), name="redirect_user"), - url(r"^guestaccess/$", GuestRedirectView.as_view(), name="guest"), - url(r"^unsupported/$", UnsupportedBrowserView.as_view(), name="unsupported"), - url(r"^$", RootURLRedirectView.as_view(), name="root_redirect"), - url(r"^", include(router.urls)), + re_path(r"^i18n/setlang/$", set_language, name="set_language"), + re_path(r"^logout/$", logout_view, name="logout"), + re_path(r"^redirectuser/$", RootURLRedirectView.as_view(), name="redirect_user"), + re_path(r"^guestaccess/$", GuestRedirectView.as_view(), name="guest"), + re_path(r"^unsupported/$", UnsupportedBrowserView.as_view(), name="unsupported"), + re_path(r"^$", RootURLRedirectView.as_view(), name="root_redirect"), + re_path(r"^", include(router.urls)), ] core_urlpatterns = ( [ - url(r"^api/", include("kolibri.core.api_urls")), - url(r"", include(i18n_patterns(lang_prefixed_patterns))), - url(r"", include("kolibri.core.content.urls")), - url(r"^status/", StatusCheckView.as_view(), name="status_check"), + re_path(r"^api/", include("kolibri.core.api_urls")), + re_path(r"", include(i18n_patterns(lang_prefixed_patterns))), + re_path(r"", include("kolibri.core.content.urls")), + re_path(r"^status/", StatusCheckView.as_view(), name="status_check"), ], "core", ) -urlpatterns = [url(r"", include(core_urlpatterns))] +urlpatterns = [re_path(r"", include(core_urlpatterns))] urlpatterns += plugin_urls() diff --git a/kolibri/core/views.py b/kolibri/core/views.py index 19e038233f9..4b59d063cc1 100644 --- a/kolibri/core/views.py +++ b/kolibri/core/views.py @@ -11,8 +11,8 @@ from django.urls import translate_url from django.utils.decorators import method_decorator from django.utils.translation import check_for_language +from django.utils.translation import gettext_lazy as _ from django.utils.translation import LANGUAGE_SESSION_KEY -from django.utils.translation import ugettext_lazy as _ from django.views.decorators.http import require_POST from django.views.generic.base import TemplateView from django.views.generic.base import View diff --git a/kolibri/deployment/default/dev_urls.py b/kolibri/deployment/default/dev_urls.py index 5aa8a21d213..3504eda9b5b 100644 --- a/kolibri/deployment/default/dev_urls.py +++ b/kolibri/deployment/default/dev_urls.py @@ -1,7 +1,7 @@ from django.conf import settings -from django.conf.urls import include -from django.conf.urls import url from django.http.response import HttpResponseRedirect +from django.urls import include +from django.urls import re_path from drf_yasg import openapi from drf_yasg.views import get_schema_view from rest_framework import permissions @@ -31,27 +31,27 @@ def webpack_redirect_view(request): ) urlpatterns = urlpatterns + [ - url(r"^__open-in-editor/", webpack_redirect_view), - url( + re_path(r"^__open-in-editor/", webpack_redirect_view), + re_path( r"^swagger(?P\.json|\.yaml)$", schema_view.without_ui(cache_timeout=0), name="schema-json", ), - url( + re_path( r"^api_explorer/$", schema_view.with_ui("swagger", cache_timeout=0), name="schema-swagger-ui", ), - url( + re_path( r"^redoc/$", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc", ), - url(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), + re_path(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), ] if getattr(settings, "DEBUG_PANEL_ACTIVE", False): import debug_toolbar - urlpatterns = [url(r"^__debug__/", include(debug_toolbar.urls))] + urlpatterns + urlpatterns = [re_path(r"^__debug__/", include(debug_toolbar.urls))] + urlpatterns diff --git a/kolibri/deployment/default/settings/base.py b/kolibri/deployment/default/settings/base.py index 8b3eb645f9c..4c2d09e736b 100644 --- a/kolibri/deployment/default/settings/base.py +++ b/kolibri/deployment/default/settings/base.py @@ -3,10 +3,10 @@ Django settings for kolibri project. For more information on this file, see -https://docs.djangoproject.com/en/1.11/topics/settings/ +https://docs.djangoproject.com/en/3.2/topics/settings/ For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.11/ref/settings/ +https://docs.djangoproject.com/en/3.2/ref/settings/ """ import os import sys @@ -50,7 +50,7 @@ LOCALE_PATHS = [os.path.join(KOLIBRI_MODULE_PATH, "locale")] # Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ +# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = "f@ey3)y^03r9^@mou97apom*+c1m#b1!cwbm50^s4yk72xce27" @@ -64,7 +64,6 @@ INSTALLED_APPS = [ "kolibri.core", - "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", @@ -137,7 +136,7 @@ # Database -# https://docs.djangoproject.com/en/1.11/ref/settings/#databases +# https://docs.djangoproject.com/en/3.2/ref/settings/#databases if conf.OPTIONS["Database"]["DATABASE_ENGINE"] == "sqlite": DATABASES = { @@ -187,9 +186,11 @@ }, } +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" + # Internationalization -# https://docs.djangoproject.com/en/1.11/topics/i18n/ +# https://docs.djangoproject.com/en/3.2/topics/i18n/ # For language names, see: # https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes @@ -326,7 +327,7 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.11/howto/static-files/ +# https://docs.djangoproject.com/en/3.2/howto/static-files/ path_prefix = conf.OPTIONS["Deployment"]["URL_PATH_PREFIX"] @@ -342,19 +343,19 @@ "django.core.files.uploadhandler.TemporaryFileUploadHandler", ] -# https://docs.djangoproject.com/en/1.11/ref/settings/#csrf-cookie-path +# https://docs.djangoproject.com/en/3.2/ref/settings/#csrf-cookie-path # Ensure that our CSRF cookie does not collide with other CSRF cookies # set by other Django apps served from the same domain. CSRF_COOKIE_PATH = path_prefix CSRF_COOKIE_NAME = "kolibri_csrftoken" -# https://docs.djangoproject.com/en/1.11/ref/settings/#session-cookie-path +# https://docs.djangoproject.com/en/3.2/ref/settings/#session-cookie-path # Ensure that our session cookie does not collidge with other session cookies # set by other Django apps served from the same domain. SESSION_COOKIE_PATH = path_prefix -# https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-LOGGING -# https://docs.djangoproject.com/en/1.11/topics/logging/ +# https://docs.djangoproject.com/en/3.2/ref/settings/#std:setting-LOGGING +# https://docs.djangoproject.com/en/3.2/topics/logging/ LOGGING = get_logging_config( conf.LOG_ROOT, @@ -364,7 +365,7 @@ # Customizing Django auth system -# https://docs.djangoproject.com/en/1.11/topics/auth/customizing/ +# https://docs.djangoproject.com/en/3.2/topics/auth/customizing/ AUTH_USER_MODEL = "kolibriauth.FacilityUser" @@ -388,16 +389,10 @@ } # System warnings to disable -# see https://docs.djangoproject.com/en/1.11/ref/settings/#silenced-system-checks +# see https://docs.djangoproject.com/en/3.2/ref/settings/#silenced-system-checks +# and https://docs.djangoproject.com/en/3.2/ref/checks/#auth SILENCED_SYSTEM_CHECKS = ["auth.W004"] -# Configuration for Django JS Reverse -# https://github.com/ierror/django-js-reverse#options - -JS_REVERSE_EXCLUDE_NAMESPACES = ["admin"] - -ENABLE_DATA_BOOTSTRAPPING = True - # Session configuration SESSION_ENGINE = "django.contrib.sessions.backends.file" diff --git a/kolibri/deployment/default/urls.py b/kolibri/deployment/default/urls.py index 1c12c4d9b2b..86e63cc0ad2 100644 --- a/kolibri/deployment/default/urls.py +++ b/kolibri/deployment/default/urls.py @@ -2,24 +2,24 @@ """kolibri URL Configuration The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/1.11/topics/http/urls/ + https://docs.djangoproject.com/en/3.2/topics/http/urls/ Examples: Function views 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') + 2. Add a URL to urlpatterns: path('', views.home, name='home') Class-based views 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') Including another URLconf 1. Add an import: from blog import urls as blog_urls - 2. Add a URL to urlpatterns: url(r'^blog/', include(blog_urls)) + 2. Add a URL to urlpatterns: path('blog/', include(blog_urls)) .. moduleauthor:: Learning Equality """ -from django.conf.urls import include -from django.conf.urls import url -from django.contrib import admin +from django.urls import include +from django.urls import path +from django.urls import re_path from morango import urls as morango_urls from kolibri.plugins.utils.urls import get_root_urls @@ -31,10 +31,9 @@ path_prefix = "" url_patterns_prefixed = [ - url(r"^admin/", admin.site.urls), - url(r"", include(morango_urls)), - url(r"", include("kolibri.core.urls")), - url(r"", include(get_root_urls())), + re_path(r"", include(morango_urls)), + re_path(r"", include("kolibri.core.urls")), + re_path(r"", include(get_root_urls())), ] -urlpatterns = [url(path_prefix, include(url_patterns_prefixed))] +urlpatterns = [path(path_prefix, include(url_patterns_prefixed))] diff --git a/kolibri/deployment/default/wsgi.py b/kolibri/deployment/default/wsgi.py index aa83001f87d..800abdad67e 100644 --- a/kolibri/deployment/default/wsgi.py +++ b/kolibri/deployment/default/wsgi.py @@ -4,7 +4,7 @@ It exposes the WSGI callable as a module-level variable named ``application``. For more information on this file, see -https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/ +https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ """ import logging import os diff --git a/kolibri/plugins/app/api.py b/kolibri/plugins/app/api.py index 73cc0e3cb5a..a079b8101b2 100644 --- a/kolibri/plugins/app/api.py +++ b/kolibri/plugins/app/api.py @@ -60,7 +60,7 @@ def get(self, request, token): if not valid_app_key(token): raise PermissionDenied("You have provided an invalid token") auth_token = request.GET.get("auth_token") - if request.user.is_anonymous() and device_provisioned() and auth_token: + if request.user.is_anonymous and device_provisioned() and auth_token: # If we are in app context, then login as the automatically created OS User try: user = FacilityUser.objects.get_or_create_os_user(auth_token) diff --git a/kolibri/plugins/app/api_urls.py b/kolibri/plugins/app/api_urls.py index b7d5bdcb1a6..af248231d35 100644 --- a/kolibri/plugins/app/api_urls.py +++ b/kolibri/plugins/app/api_urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from rest_framework import routers from .api import AppCommandsViewset @@ -10,6 +10,8 @@ router.register(r"appcommands", AppCommandsViewset, basename="appcommands") urlpatterns = [ - url(r"^", include(router.urls)), - url(r"^initialize/([0-9a-f]{32})", InitializeAppView.as_view(), name="initialize"), + re_path(r"^", include(router.urls)), + re_path( + r"^initialize/([0-9a-f]{32})$", InitializeAppView.as_view(), name="initialize" + ), ] diff --git a/kolibri/plugins/coach/api.py b/kolibri/plugins/coach/api.py index 008a0df6bc4..c589cf2c14c 100644 --- a/kolibri/plugins/coach/api.py +++ b/kolibri/plugins/coach/api.py @@ -180,7 +180,10 @@ def get_queryset(self): :param: after integer: all the notifications after this id will be sent. :param: limit integer: sets the number of notifications to provide """ - classroom_id = self.kwargs["classroom_id"] + classroom_id = self.kwargs.get("classroom_id", None) + + if classroom_id is None: + return LearnerProgressNotification.objects.none() notifications_query = LearnerProgressNotification.objects.filter( classroom_id=classroom_id diff --git a/kolibri/plugins/coach/api_urls.py b/kolibri/plugins/coach/api_urls.py index 32fa53f2ab7..63f71686e51 100644 --- a/kolibri/plugins/coach/api_urls.py +++ b/kolibri/plugins/coach/api_urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from rest_framework import routers from .api import ClassroomNotificationsViewset @@ -30,4 +30,4 @@ basename="practicequizdifficulties", ) -urlpatterns = [url(r"^", include(router.urls))] +urlpatterns = [re_path(r"^", include(router.urls))] diff --git a/kolibri/plugins/coach/kolibri_plugin.py b/kolibri/plugins/coach/kolibri_plugin.py index 9e854c47827..7dc8327f89c 100644 --- a/kolibri/plugins/coach/kolibri_plugin.py +++ b/kolibri/plugins/coach/kolibri_plugin.py @@ -7,7 +7,7 @@ from kolibri.plugins import KolibriPluginBase from kolibri.plugins.hooks import register_hook from kolibri.utils import translation -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ logger = logging.getLogger(__name__) diff --git a/kolibri/plugins/coach/test/test_class_summary.py b/kolibri/plugins/coach/test/test_class_summary.py index 462738ad344..de21baff8a2 100644 --- a/kolibri/plugins/coach/test/test_class_summary.py +++ b/kolibri/plugins/coach/test/test_class_summary.py @@ -17,6 +17,8 @@ class ClassSummaryTestCase(EvaluationMixin, APITestCase): + databases = "__all__" + fixtures = ["content_test.json"] the_channel_id = "6199dde695db4ee4ab392222d5af1e5c" @@ -203,6 +205,8 @@ def test_practice_quiz_summary(self): class ClassSummaryDiffTestCase(EvaluationMixin, APITestCase): + databases = "__all__" + def test_practice_quiz_summary(self): provision_device() classroom = Classroom.objects.create(name="classrom", parent=self.facility) diff --git a/kolibri/plugins/coach/test/test_classroom_notifications.py b/kolibri/plugins/coach/test/test_classroom_notifications.py index 665e32c02af..d92af9270e1 100644 --- a/kolibri/plugins/coach/test/test_classroom_notifications.py +++ b/kolibri/plugins/coach/test/test_classroom_notifications.py @@ -10,6 +10,8 @@ class ClassroomNotificationsTestCase(APITestCase): + databases = "__all__" + def setUp(self): provision_device() self.facility = Facility.objects.create(name="My Facility") diff --git a/kolibri/plugins/coach/test/test_difficult_questions.py b/kolibri/plugins/coach/test/test_difficult_questions.py index 6106b83e819..883edd1b308 100644 --- a/kolibri/plugins/coach/test/test_difficult_questions.py +++ b/kolibri/plugins/coach/test/test_difficult_questions.py @@ -1,4 +1,3 @@ -import json from datetime import timedelta from django.urls import reverse @@ -78,15 +77,13 @@ def setUp(self): title="My Lesson", created_by=self.facility_and_classroom_coach, collection=self.classroom, - resources=json.dumps( - [ - { - "contentnode_id": self.node_1.id, - "content_id": self.node_1.content_id, - "channel_id": self.channel_id, - } - ] - ), + resources=[ + { + "contentnode_id": self.node_1.id, + "content_id": self.node_1.content_id, + "channel_id": self.channel_id, + } + ], ) self.assignment_1 = LessonAssignment.objects.create( lesson=self.lesson, @@ -820,15 +817,13 @@ def setUp(self): title="My Lesson", created_by=self.facility_and_classroom_coach, collection=self.classroom, - resources=json.dumps( - [ - { - "contentnode_id": self.node_1.id, - "content_id": self.node_1.content_id, - "channel_id": self.channel_id, - } - ] - ), + resources=[ + { + "contentnode_id": self.node_1.id, + "content_id": self.node_1.content_id, + "channel_id": self.channel_id, + } + ], ) self.assignment_1 = LessonAssignment.objects.create( lesson=self.lesson, diff --git a/kolibri/plugins/coach/test/test_lesson_report.py b/kolibri/plugins/coach/test/test_lesson_report.py index 8c5982d2fdb..6238b7a7877 100644 --- a/kolibri/plugins/coach/test/test_lesson_report.py +++ b/kolibri/plugins/coach/test/test_lesson_report.py @@ -1,5 +1,4 @@ import datetime -import json from django.urls import reverse from rest_framework.test import APITestCase @@ -91,15 +90,13 @@ def setUp(self): title="My Lesson", created_by=self.facility_and_classroom_coach, collection=self.classroom, - resources=json.dumps( - [ - { - "contentnode_id": self.node_1.id, - "content_id": self.node_1.content_id, - "channel_id": self.channel_id, - } - ] - ), + resources=[ + { + "contentnode_id": self.node_1.id, + "content_id": self.node_1.content_id, + "channel_id": self.channel_id, + } + ], ) self.assignment_1 = LessonAssignment.objects.create( lesson=self.lesson, diff --git a/kolibri/plugins/coach/urls.py b/kolibri/plugins/coach/urls.py index 410069db6f6..514853b895f 100644 --- a/kolibri/plugins/coach/urls.py +++ b/kolibri/plugins/coach/urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import url +from django.urls import re_path from . import views -urlpatterns = [url(r"^$", views.CoachView.as_view(), name="coach")] +urlpatterns = [re_path(r"^$", views.CoachView.as_view(), name="coach")] diff --git a/kolibri/plugins/device/api.py b/kolibri/plugins/device/api.py index 4a09c4e12cd..108e00b443f 100644 --- a/kolibri/plugins/device/api.py +++ b/kolibri/plugins/device/api.py @@ -99,7 +99,7 @@ class Meta: class DeviceChannelMetadataViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = DeviceChannelMetadataSerializer filter_backends = (DjangoFilterBackend,) - filter_class = DeviceChannelMetadataFilter + filterset_class = DeviceChannelMetadataFilter permission_classes = (CanManageContent,) def get_queryset(self): diff --git a/kolibri/plugins/device/api_urls.py b/kolibri/plugins/device/api_urls.py index 7c1f6bc4694..126053f160d 100644 --- a/kolibri/plugins/device/api_urls.py +++ b/kolibri/plugins/device/api_urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from rest_framework import routers from .api import CalculateImportExportSizeView @@ -13,13 +13,13 @@ ) urlpatterns = [ - url(r"^", include(router.urls)), - url( + re_path(r"^", include(router.urls)), + re_path( r"devicechannelorder", DeviceChannelOrderView.as_view(), name="devicechannelorder", ), - url( + re_path( r"importexportsizeview", CalculateImportExportSizeView.as_view(), name="importexportsizeview", diff --git a/kolibri/plugins/device/urls.py b/kolibri/plugins/device/urls.py index 40e3625a238..a3fceb0f777 100644 --- a/kolibri/plugins/device/urls.py +++ b/kolibri/plugins/device/urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import url +from django.urls import re_path from .views import DeviceManagementView -urlpatterns = [url(r"^$", DeviceManagementView.as_view(), name="device_management")] +urlpatterns = [re_path(r"^$", DeviceManagementView.as_view(), name="device_management")] diff --git a/kolibri/plugins/facility/api_urls.py b/kolibri/plugins/facility/api_urls.py index 21e8162714b..68d5fff762e 100644 --- a/kolibri/plugins/facility/api_urls.py +++ b/kolibri/plugins/facility/api_urls.py @@ -1,21 +1,21 @@ -from django.conf.urls import url +from django.urls import re_path from .views import download_csv_file from .views import exported_csv_info from .views import first_log_date urlpatterns = [ - url( + re_path( r"^downloadcsvfile/(?P.*)/(?P.*)/$", download_csv_file, name="download_csv_file", ), - url( + re_path( r"^exportedcsvinfo/(?P.*)/$", exported_csv_info, name="exportedcsvinfo", ), - url( + re_path( r"^firstlogdate/(?P.*)/$", first_log_date, name="firstlogdate", diff --git a/kolibri/plugins/facility/kolibri_plugin.py b/kolibri/plugins/facility/kolibri_plugin.py index 85c0a19c9c4..32b61530d57 100644 --- a/kolibri/plugins/facility/kolibri_plugin.py +++ b/kolibri/plugins/facility/kolibri_plugin.py @@ -5,7 +5,7 @@ from kolibri.plugins import KolibriPluginBase from kolibri.plugins.hooks import register_hook from kolibri.utils import translation -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ class FacilityManagementPlugin(KolibriPluginBase): diff --git a/kolibri/plugins/facility/urls.py b/kolibri/plugins/facility/urls.py index 985578aa7d9..95d20a13f41 100644 --- a/kolibri/plugins/facility/urls.py +++ b/kolibri/plugins/facility/urls.py @@ -1,5 +1,7 @@ -from django.conf.urls import url +from django.urls import re_path from .views import FacilityManagementView -urlpatterns = [url(r"^$", FacilityManagementView.as_view(), name="facility_management")] +urlpatterns = [ + re_path(r"^$", FacilityManagementView.as_view(), name="facility_management") +] diff --git a/kolibri/plugins/facility/views.py b/kolibri/plugins/facility/views.py index 9aa8f091e77..8960ec75284 100644 --- a/kolibri/plugins/facility/views.py +++ b/kolibri/plugins/facility/views.py @@ -216,17 +216,17 @@ def download_csv_file(request, csv_type, facility_id): # generate a file response response = FileResponse(io.open(filepath, "rb")) # set the content-type by guessing from the filename - response["Content-Type"] = "text/csv" + response.headers["Content-Type"] = "text/csv" # set the content-disposition as attachment to force download if csv_type == "user": - response["Content-Disposition"] = "attachment; filename={}".format( + response.headers["Content-Disposition"] = "attachment; filename={}".format( str(csv_translated_filenames[csv_type]).format( facility.name, facility.id[:4] ) ) else: - response["Content-Disposition"] = "attachment; filename={}".format( + response.headers["Content-Disposition"] = "attachment; filename={}".format( str(csv_translated_filenames[csv_type]).format( facility.name, facility.id[:4], start[:10], end[:10] ) @@ -234,6 +234,6 @@ def download_csv_file(request, csv_type, facility_id): translation.deactivate() # set the content-length to the file size - response["Content-Length"] = os.path.getsize(filepath) + response.headers["Content-Length"] = os.path.getsize(filepath) return response diff --git a/kolibri/plugins/learn/api_urls.py b/kolibri/plugins/learn/api_urls.py index a1689fa481b..441d0512abe 100644 --- a/kolibri/plugins/learn/api_urls.py +++ b/kolibri/plugins/learn/api_urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from rest_framework import routers from .viewsets import LearnerClassroomViewset @@ -15,7 +15,7 @@ urlpatterns = [ - url(r"^", include(router.urls)), - url(r"state", LearnStateView.as_view(), name="state"), - url(r"homehydrate", LearnHomePageHydrationView.as_view(), name="homehydrate"), + re_path(r"^", include(router.urls)), + re_path(r"state", LearnStateView.as_view(), name="state"), + re_path(r"homehydrate", LearnHomePageHydrationView.as_view(), name="homehydrate"), ] diff --git a/kolibri/plugins/learn/kolibri_plugin.py b/kolibri/plugins/learn/kolibri_plugin.py index 9e19709626d..4d14875b3e7 100644 --- a/kolibri/plugins/learn/kolibri_plugin.py +++ b/kolibri/plugins/learn/kolibri_plugin.py @@ -17,7 +17,7 @@ from kolibri.plugins.hooks import register_hook from kolibri.utils import conf from kolibri.utils import translation -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ class Learn(KolibriPluginBase): diff --git a/kolibri/plugins/learn/test/test_hooks.py b/kolibri/plugins/learn/test/test_hooks.py index 3a9d55e819f..054aab727fa 100644 --- a/kolibri/plugins/learn/test/test_hooks.py +++ b/kolibri/plugins/learn/test/test_hooks.py @@ -6,6 +6,8 @@ class NetworkDiscoveryForSoUDHookTestCase(TestCase): + databases = "__all__" + def setUp(self): super(NetworkDiscoveryForSoUDHookTestCase, self).setUp() self.hook = NetworkLocationDiscoveryHook.get_hook( diff --git a/kolibri/plugins/learn/urls.py b/kolibri/plugins/learn/urls.py index 2e89a21203b..a7e420ac8f8 100644 --- a/kolibri/plugins/learn/urls.py +++ b/kolibri/plugins/learn/urls.py @@ -1,9 +1,9 @@ -from django.conf.urls import url +from django.urls import re_path from .views import LearnView from .views import MyDownloadsView urlpatterns = [ - url(r"^$", LearnView.as_view(), name="learn"), - url(r"^my-downloads$", MyDownloadsView.as_view(), name="my_downloads"), + re_path(r"^$", LearnView.as_view(), name="learn"), + re_path(r"^my-downloads$", MyDownloadsView.as_view(), name="my_downloads"), ] diff --git a/kolibri/plugins/policies/kolibri_plugin.py b/kolibri/plugins/policies/kolibri_plugin.py index d3fd61521dc..9a48a9989b4 100644 --- a/kolibri/plugins/policies/kolibri_plugin.py +++ b/kolibri/plugins/policies/kolibri_plugin.py @@ -2,7 +2,7 @@ from kolibri.plugins import KolibriPluginBase from kolibri.plugins.hooks import register_hook from kolibri.utils import translation -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ class Policies(KolibriPluginBase): diff --git a/kolibri/plugins/policies/urls.py b/kolibri/plugins/policies/urls.py index a1607902f5b..9c016ea6ecf 100644 --- a/kolibri/plugins/policies/urls.py +++ b/kolibri/plugins/policies/urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import url +from django.urls import re_path from . import views -urlpatterns = [url(r"^$", views.PoliciesView.as_view(), name="policies")] +urlpatterns = [re_path(r"^$", views.PoliciesView.as_view(), name="policies")] diff --git a/kolibri/plugins/pwa/root_urls.py b/kolibri/plugins/pwa/root_urls.py index dd8fc8b11d6..3a0d29fc3b2 100644 --- a/kolibri/plugins/pwa/root_urls.py +++ b/kolibri/plugins/pwa/root_urls.py @@ -3,7 +3,7 @@ # # Copyright 2023 Endless OS Foundation, LLC # SPDX-License-Identifier: MIT -from django.conf.urls import url +from django.urls import re_path from .views import PwaServiceWorkerView @@ -18,5 +18,5 @@ # namespaces are not merged. Having two urlpattern lists with identical # namespaces breaks name reverse lookup. urlpatterns = [ - url(r"^sw.js$", PwaServiceWorkerView.as_view(), name="pwa_service_worker"), + re_path(r"^sw.js$", PwaServiceWorkerView.as_view(), name="pwa_service_worker"), ] diff --git a/kolibri/plugins/pwa/urls.py b/kolibri/plugins/pwa/urls.py index f01f369afa8..e0e8e034f1b 100644 --- a/kolibri/plugins/pwa/urls.py +++ b/kolibri/plugins/pwa/urls.py @@ -3,10 +3,10 @@ # # Copyright 2023 Endless OS Foundation, LLC # SPDX-License-Identifier: MIT -from django.conf.urls import url +from django.urls import re_path from .views import PwaManifestView urlpatterns = [ - url(r"^manifest.webmanifest$", PwaManifestView.as_view(), name="manifest"), + re_path(r"^manifest.webmanifest$", PwaManifestView.as_view(), name="manifest"), ] diff --git a/kolibri/plugins/setup_wizard/kolibri_plugin.py b/kolibri/plugins/setup_wizard/kolibri_plugin.py index 9d96505770c..b560975fbd8 100644 --- a/kolibri/plugins/setup_wizard/kolibri_plugin.py +++ b/kolibri/plugins/setup_wizard/kolibri_plugin.py @@ -3,7 +3,7 @@ from kolibri.plugins import KolibriPluginBase from kolibri.plugins.hooks import register_hook from kolibri.utils import translation -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ class SetupWizardPlugin(KolibriPluginBase): diff --git a/kolibri/plugins/setup_wizard/test/test_api.py b/kolibri/plugins/setup_wizard/test/test_api.py index 50171f0a327..51571862ba4 100644 --- a/kolibri/plugins/setup_wizard/test/test_api.py +++ b/kolibri/plugins/setup_wizard/test/test_api.py @@ -34,6 +34,8 @@ def test_only_returns_admins(self): class GrantSuperuserPermissionsTest(APITestCase): + databases = "__all__" + def setUp(self): clear_process_cache() diff --git a/kolibri/plugins/setup_wizard/urls.py b/kolibri/plugins/setup_wizard/urls.py index 4aa52f28b37..342b12c670c 100644 --- a/kolibri/plugins/setup_wizard/urls.py +++ b/kolibri/plugins/setup_wizard/urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import url +from django.urls import re_path from . import views -urlpatterns = [url(r"^$", views.SetupWizardView.as_view(), name="setupwizard")] +urlpatterns = [re_path(r"^$", views.SetupWizardView.as_view(), name="setupwizard")] diff --git a/kolibri/plugins/user_auth/root_urls.py b/kolibri/plugins/user_auth/root_urls.py index 3cb8b156ffd..708b669ab00 100644 --- a/kolibri/plugins/user_auth/root_urls.py +++ b/kolibri/plugins/user_auth/root_urls.py @@ -1,14 +1,14 @@ """ This is here to enable redirects from the old /user endpoint to /auth """ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from django.views.generic.base import RedirectView from kolibri.core.device.translation import i18n_patterns redirect_patterns = [ - url( + re_path( r"^user/$", RedirectView.as_view( pattern_name="kolibri:kolibri.plugins.user_auth:user_auth", permanent=True @@ -17,4 +17,4 @@ ), ] -urlpatterns = [url(r"", include(i18n_patterns(redirect_patterns)))] +urlpatterns = [re_path(r"", include(i18n_patterns(redirect_patterns)))] diff --git a/kolibri/plugins/user_auth/urls.py b/kolibri/plugins/user_auth/urls.py index 2216a71074c..811d76940a2 100644 --- a/kolibri/plugins/user_auth/urls.py +++ b/kolibri/plugins/user_auth/urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import url +from django.urls import re_path from . import views -urlpatterns = [url(r"^$", views.UserAuthView.as_view(), name="user_auth")] +urlpatterns = [re_path(r"^$", views.UserAuthView.as_view(), name="user_auth")] diff --git a/kolibri/plugins/user_profile/api_urls.py b/kolibri/plugins/user_profile/api_urls.py index 22cfe1fd5b6..04d193e696f 100644 --- a/kolibri/plugins/user_profile/api_urls.py +++ b/kolibri/plugins/user_profile/api_urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import url +from django.urls import re_path from .viewsets import LoginMergedUserViewset from .viewsets import OnMyOwnSetupViewset @@ -6,18 +6,18 @@ from .viewsets import RemoteFacilityUserViewset urlpatterns = [ - url(r"onmyownsetup", OnMyOwnSetupViewset.as_view(), name="onmyownsetup"), - url( + re_path(r"onmyownsetup", OnMyOwnSetupViewset.as_view(), name="onmyownsetup"), + re_path( r"remotefacilityuser", RemoteFacilityUserViewset.as_view(), name="remotefacilityuser", ), - url( + re_path( r"remotefacilityauthenticateduserinfo", RemoteFacilityUserAuthenticatedViewset.as_view(), name="remotefacilityauthenticateduserinfo", ), - url( + re_path( r"loginmergeduser", LoginMergedUserViewset.as_view(), name="loginmergeduser", diff --git a/kolibri/plugins/user_profile/kolibri_plugin.py b/kolibri/plugins/user_profile/kolibri_plugin.py index 44eb94327cd..74f361404da 100644 --- a/kolibri/plugins/user_profile/kolibri_plugin.py +++ b/kolibri/plugins/user_profile/kolibri_plugin.py @@ -3,7 +3,7 @@ from kolibri.plugins import KolibriPluginBase from kolibri.plugins.hooks import register_hook from kolibri.utils import translation -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ class UserProfile(KolibriPluginBase): diff --git a/kolibri/plugins/user_profile/tasks.py b/kolibri/plugins/user_profile/tasks.py index c16d84422d5..44ec4715c39 100644 --- a/kolibri/plugins/user_profile/tasks.py +++ b/kolibri/plugins/user_profile/tasks.py @@ -25,7 +25,7 @@ from kolibri.core.tasks.permissions import PermissionsFromAny from kolibri.core.tasks.utils import get_current_job from kolibri.core.utils.urls import reverse_remote -from kolibri.utils.translation import ugettext as _ +from kolibri.utils.translation import gettext as _ class MergeUserValidator(PeerImportSingleSyncJobValidator): @@ -53,7 +53,7 @@ def validate(self, data): # If we didn't break out of the loop, then we need to raise the original error raise - job_data["kwargs"]["local_user_id"] = data["local_user_id"].id + job_data["args"].append(data["local_user_id"].id) job_data["extra_metadata"].update(user_fullname=data["local_user_id"].full_name) if data.get("new_superuser_id"): job_data["kwargs"]["new_superuser_id"] = data["new_superuser_id"].id @@ -120,7 +120,9 @@ def start_soud_sync(user_id): ], status_fn=status_fn, ) -def mergeuser(command, **kwargs): +def mergeuser( + command, local_user_id, new_superuser_id=None, set_as_super_user=False, **kwargs +): """ This is an example of the POST payload to create this task: { @@ -138,7 +140,6 @@ def mergeuser(command, **kwargs): this task will try to create the user. """ - local_user_id = kwargs.pop("local_user_id") local_user = FacilityUser.objects.get(id=local_user_id) job = get_current_job() @@ -165,7 +166,6 @@ def mergeuser(command, **kwargs): # so that the job has to be retried by the user. raise - new_superuser_id = kwargs.get("new_superuser_id") if new_superuser_id and local_user.is_superuser: new_superuser = FacilityUser.objects.get(id=new_superuser_id) # make the user a new super user for this device: @@ -176,7 +176,7 @@ def mergeuser(command, **kwargs): # create token to validate user in the new facility # after it's deleted in the current facility: - remote_user_pk = job.kwargs["user"] + remote_user_pk = kwargs["user"] remote_user = FacilityUser.objects.get(pk=remote_user_pk) token = TokenGenerator().make_token(remote_user) job.extra_metadata["token"] = token diff --git a/kolibri/plugins/user_profile/urls.py b/kolibri/plugins/user_profile/urls.py index 59e0fe94b78..df0b747b95f 100644 --- a/kolibri/plugins/user_profile/urls.py +++ b/kolibri/plugins/user_profile/urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import url +from django.urls import re_path from . import views -urlpatterns = [url(r"^$", views.UserProfileView.as_view(), name="user_profile")] +urlpatterns = [re_path(r"^$", views.UserProfileView.as_view(), name="user_profile")] diff --git a/kolibri/plugins/utils/urls.py b/kolibri/plugins/utils/urls.py index 513c778f950..31fc7f01899 100644 --- a/kolibri/plugins/utils/urls.py +++ b/kolibri/plugins/utils/urls.py @@ -1,7 +1,7 @@ import logging -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import path from kolibri.core.device.translation import i18n_patterns from kolibri.plugins.registry import registered_plugins @@ -66,10 +66,10 @@ def get_urls(): if url_module: instance_patterns += i18n_patterns(url_module.urlpatterns, prefix=slug) if api_url_module: - instance_patterns.append(url(slug + "api/", include(api_url_module))) + instance_patterns.append(path(slug + "api/", include(api_url_module))) if instance_patterns: urlpatterns.append( - url( + path( "", include((instance_patterns, plugin_instance.module_path)), ) @@ -83,6 +83,6 @@ def get_root_urls(): for plugin_instance in registered_plugins: root_url_module = plugin_instance.root_url_module if root_url_module: - urlpatterns.append(url("", include(root_url_module))) + urlpatterns.append(path("", include(root_url_module))) return urlpatterns diff --git a/kolibri/utils/compat.py b/kolibri/utils/compat.py index 7ec084bbf75..0285bd952be 100644 --- a/kolibri/utils/compat.py +++ b/kolibri/utils/compat.py @@ -1,7 +1,6 @@ """ Compatibility layer for Python 2+3 """ -import sys from importlib.util import find_spec @@ -14,68 +13,3 @@ def module_exists(module_path): return find_spec(module_path) is not None except ImportError: return False - - -def monkey_patch_collections(): - """ - Monkey-patching for the collections module is required for Python 3.10 - and above. - Prior to 3.10, the collections module still contained all the entities defined in - collections.abc from Python 3.3 onwards. Here we patch those back into main - collections module. - This can be removed when we upgrade to a version of Django that is Python 3.10 compatible. - """ - if sys.version_info < (3, 10): - return - import collections - from collections import abc - - for name in dir(abc): - if not hasattr(collections, name): - setattr(collections, name, getattr(abc, name)) - - -def monkey_patch_translation(): - """ - Monkey-patching for the gettext module is required for Python 3.11 - and above. - Prior to 3.11, the gettext module classes still had the deprecated set_output_charset - This can be removed when we upgrade to a version of Django that no longer relies - on this deprecated Python 2.7 only call. - """ - if sys.version_info < (3, 11): - return - - import gettext - - def set_output_charset(*args, **kwargs): - pass - - gettext.NullTranslations.set_output_charset = set_output_charset - - original_translation = gettext.translation - - def translation( - domain, - localedir=None, - languages=None, - class_=None, - fallback=False, - codeset=None, - ): - return original_translation( - domain, - localedir=localedir, - languages=languages, - class_=class_, - fallback=fallback, - ) - - gettext.translation = translation - - original_install = gettext.install - - def install(domain, localedir=None, codeset=None, names=None): - return original_install(domain, localedir=localedir, names=names) - - gettext.install = install diff --git a/kolibri/utils/env.py b/kolibri/utils/env.py index 8e37c769c81..cdd77c34b13 100644 --- a/kolibri/utils/env.py +++ b/kolibri/utils/env.py @@ -3,9 +3,6 @@ import platform import sys -from kolibri.utils.compat import monkey_patch_collections -from kolibri.utils.compat import monkey_patch_translation - def settings_module(): from .build_config.default_settings import settings_path @@ -112,10 +109,6 @@ def set_env(): from kolibri import dist as kolibri_dist # noqa - monkey_patch_collections() - - monkey_patch_translation() - sys.path = [os.path.realpath(os.path.dirname(kolibri_dist.__file__))] + sys.path # Add path for c extensions to sys.path diff --git a/kolibri/utils/translation.py b/kolibri/utils/translation.py index 362aefe596d..31beda19492 100644 --- a/kolibri/utils/translation.py +++ b/kolibri/utils/translation.py @@ -2,18 +2,18 @@ This module is used to provide translation support for Kolibri, prior to the loading of the Django stack. Most of these functions are vendored from Django: -https://github.com/django/django/blob/stable/1.11.x/django/utils/translation/trans_real.py +https://github.com/django/django/blob/stable/3.2.x/django/utils/translation/trans_real.py In order to give a completely transparent interface. """ import gettext as gettext_module import os -from threading import local +from contextlib import ContextDecorator +from asgiref.local import Local from django.core.exceptions import AppRegistryNotReady from django.core.exceptions import ImproperlyConfigured from django.utils import translation as django_translation_module -from django.utils.decorators import ContextDecorator from django.utils.safestring import mark_safe from django.utils.safestring import SafeData from django.utils.translation.trans_real import DjangoTranslation @@ -24,7 +24,7 @@ # Translations are cached in a dictionary for every language. # The active translations are stored by threadid to make them thread local. _translations = {} -_active = local() +_active = Local() _default = "en" @@ -127,40 +127,27 @@ def __exit__(self, exc_type, exc_value, traceback): activate(self.old_language) -def do_translate(message, translation_function): +@prefer_django +def gettext(message): """ - Translates 'message' using the given 'translation_function' name -- which - will be either gettext or ugettext. It uses the current thread to find the + Translate the 'message' string. It uses the current thread to find the translation object to use. If no current translation is activated, the message will be run through the default translation object. """ global _default - # str() is allowing a bytestring message to remain bytestring on Python 2 - eol_message = message.replace(str("\r\n"), str("\n")).replace(str("\r"), str("\n")) + eol_message = message.replace("\r\n", "\n").replace("\r", "\n") + + if eol_message: + translation_object = getattr(_active, "value", _default) - if len(eol_message) == 0: + result = translation_object.gettext(eol_message) + else: # Returns an empty value of the corresponding type if an empty message # is given, instead of metadata, which is the default gettext behavior. result = type(message)("") - else: - translation_object = getattr(_active, "value", _default) - - result = getattr(translation_object, translation_function)(eol_message) if isinstance(message, SafeData): return mark_safe(result) return result - - -@prefer_django -def gettext(message): - """ - Returns a string of the translation of the message. - Returns a string on Python 3 and an UTF-8-encoded bytestring on Python 2. - """ - return do_translate(message, "gettext") - - -ugettext = gettext diff --git a/requirements/base.txt b/requirements/base.txt index 9e7700517f9..bb1c71ab0e0 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,26 +1,25 @@ diskcache==4.1.0 -django-js-asset<2 -django-filter==1.1.0 # pyup: <2.0.0 -django-js-reverse==0.9.1 -djangorestframework==3.9.1 -django==1.11.29 # pyup: >=1.11,<2 +django-filter==21.1 +django-js-reverse==0.10.2 +djangorestframework==3.14.0 +django==3.2.25 colorlog==3.2.0 # pyup: <4.0.0 configobj==5.0.6 -django-mptt==0.9.1 +django-mptt==0.14.0 requests==2.27.1 cheroot==8.6.0 magicbus==4.1.2 le-utils==0.2.2 -jsonfield==2.0.2 -morango==0.7.1 +jsonfield==3.1.0 +morango==0.8.0 tzlocal==2.1 pytz==2022.1 python-dateutil==2.8.2 sqlalchemy==1.4.49 semver==2.8.1 -django-redis-cache==2.0.0 +django-redis-cache==3.0.1 redis==3.2.1 -html5lib==1.0.1 +html5lib==1.1 zeroconf-py2compat==0.19.17 Click==7.0 whitenoise==4.1.4 diff --git a/requirements/cext.txt b/requirements/cext.txt index da4bb82ad31..9ab3495d652 100644 --- a/requirements/cext.txt +++ b/requirements/cext.txt @@ -4,5 +4,5 @@ # in other files to ensure they are properly # installed - as these dependencies are installed # without their dependencies. -cffi==1.14.4 -cryptography==3.3.2 +cffi==1.15.1 +cryptography==40.0.2 diff --git a/requirements/cext_noarch.txt b/requirements/cext_noarch.txt index 640c1563941..9d9a0b7ec4f 100644 --- a/requirements/cext_noarch.txt +++ b/requirements/cext_noarch.txt @@ -1,6 +1,6 @@ # These are unique requirements for C extensions, but the # ones that don't need to be shipped in one version per # arch/OS/pyVesion -pyOpenSSL==18.0.0 -asn1crypto==1.4.0 +pyOpenSSL==23.2.0 +asn1crypto==1.5.1 pycparser==2.21 diff --git a/requirements/dev.txt b/requirements/dev.txt index 54e9471c0b4..01844a7e425 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -7,7 +7,7 @@ tox==3.1.3 django-debug-panel==0.8.3 django-debug-toolbar==1.9.1 ruamel.yaml.clib==0.2.2; python_version < "3" -drf-yasg==1.17.1 +drf-yasg==1.21.7 coreapi==2.3.3 nodeenv==1.3.3 sqlacodegen==2.1.0 diff --git a/requirements/postgres.txt b/requirements/postgres.txt index 00798791ad6..daf2cf494cf 100644 --- a/requirements/postgres.txt +++ b/requirements/postgres.txt @@ -1,2 +1,2 @@ # Additional reqs for running kolibri with a postgres db layer -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.9