diff --git a/kolibri/core/auth/apps.py b/kolibri/core/auth/apps.py index 5a13771accd..9513b1067b1 100644 --- a/kolibri/core/auth/apps.py +++ b/kolibri/core/auth/apps.py @@ -13,3 +13,10 @@ class KolibriAuthConfig(AppConfig): def ready(self): from .signals import cascade_delete_membership # noqa: F401 from .signals import cascade_delete_user # noqa: F401 + + from kolibri.core.auth.sync_event_hook_utils import ( + register_sync_event_handlers, + ) # noqa: F401 + from morango.api.viewsets import session_controller # noqa: F401 + + register_sync_event_handlers(session_controller) diff --git a/kolibri/core/auth/hooks.py b/kolibri/core/auth/hooks.py new file mode 100644 index 00000000000..74ceaeb249b --- /dev/null +++ b/kolibri/core/auth/hooks.py @@ -0,0 +1,29 @@ +from kolibri.plugins.hooks import define_hook +from kolibri.plugins.hooks import KolibriHook + + +@define_hook +class FacilityDataSyncHook(KolibriHook): + """ + A hook to allow plugins to register callbacks for sync events they're interested in. + """ + + def pre_transfer( + self, + dataset_id, + local_is_single_user, + remote_is_single_user, + single_user_id, + context, + ): + pass + + def post_transfer( + self, + dataset_id, + local_is_single_user, + remote_is_single_user, + single_user_id, + context, + ): + pass diff --git a/kolibri/core/auth/management/commands/sync.py b/kolibri/core/auth/management/commands/sync.py index 1e3c5ab3beb..64b03fb3204 100644 --- a/kolibri/core/auth/management/commands/sync.py +++ b/kolibri/core/auth/management/commands/sync.py @@ -23,15 +23,14 @@ from kolibri.core.auth.management.utils import get_facility from kolibri.core.auth.management.utils import run_once from kolibri.core.auth.models import dataset_cache -from kolibri.core.lessons.single_user_assignment_utils import ( - register_single_user_sync_lesson_handlers, -) +from kolibri.core.auth.sync_event_hook_utils import register_sync_event_handlers from kolibri.core.logger.utils.data import bytes_for_humans from kolibri.core.tasks.exceptions import UserCancelledError from kolibri.core.tasks.management.commands.base import AsyncCommand from kolibri.core.utils.lock import db_lock from kolibri.utils import conf + DATA_PORTAL_SYNCING_BASE_URL = conf.OPTIONS["Urls"]["DATA_PORTAL_SYNCING_BASE_URL"] TRANSFER_MESSAGE = "{records_transferred}/{records_total}, {transfer_total}" @@ -221,7 +220,7 @@ def handle_async(self, *args, **options): # noqa C901 client_cert, server_cert, chunk_size=chunk_size ) - register_single_user_sync_lesson_handlers(sync_session_client.controller) + register_sync_event_handlers(sync_session_client.controller) try: # pull from server diff --git a/kolibri/core/auth/sync_event_hook_utils.py b/kolibri/core/auth/sync_event_hook_utils.py new file mode 100644 index 00000000000..5e94a5d3f9f --- /dev/null +++ b/kolibri/core/auth/sync_event_hook_utils.py @@ -0,0 +1,69 @@ +import json + +from morango.sync.context import LocalSessionContext + +from kolibri.core.auth.constants.morango_sync import ScopeDefinitions +from kolibri.core.auth.hooks import FacilityDataSyncHook + + +def _get_our_cert(context): + ss = context.sync_session + return ss.server_certificate if ss.is_server else ss.client_certificate + + +def _get_their_cert(context): + ss = context.sync_session + return ss.client_certificate if ss.is_server else ss.server_certificate + + +def _this_side_using_single_user_cert(context): + return _get_our_cert(context).scope_definition_id == ScopeDefinitions.SINGLE_USER + + +def _other_side_using_single_user_cert(context): + return _get_their_cert(context).scope_definition_id == ScopeDefinitions.SINGLE_USER + + +def _get_user_id_for_single_user_sync(context): + if _other_side_using_single_user_cert(context): + cert = _get_their_cert(context) + elif _this_side_using_single_user_cert(context): + cert = _get_our_cert(context) + else: + return None + return json.loads(cert.scope_params)["user_id"] + + +def _extract_kwargs_from_context(context): + return { + "dataset_id": _get_our_cert(context).get_root().id, + "local_is_single_user": _this_side_using_single_user_cert(context), + "remote_is_single_user": _other_side_using_single_user_cert(context), + "single_user_id": _get_user_id_for_single_user_sync(context), + "context": context, + } + + +def _pre_transfer_handler(context): + assert context is not None + + kwargs = _extract_kwargs_from_context(context) + + if isinstance(context, LocalSessionContext): + for hook in FacilityDataSyncHook.registered_hooks: + hook.pre_transfer(**kwargs) + + +def _post_transfer_handler(context): + assert context is not None + + kwargs = _extract_kwargs_from_context(context) + + if isinstance(context, LocalSessionContext): + for hook in FacilityDataSyncHook.registered_hooks: + hook.post_transfer(**kwargs) + + +def register_sync_event_handlers(session_controller): + session_controller.signals.initializing.completed.connect(_pre_transfer_handler) + session_controller.signals.cleanup.completed.connect(_post_transfer_handler) diff --git a/kolibri/core/auth/test/test_morango_integration.py b/kolibri/core/auth/test/test_morango_integration.py index dbe128de952..8942808dcac 100644 --- a/kolibri/core/auth/test/test_morango_integration.py +++ b/kolibri/core/auth/test/test_morango_integration.py @@ -639,7 +639,7 @@ def test_single_user_assignment_sync(self, servers): # repeat the same sets of scenarios, but separately for an exam and a lesson, and with # different methods for disabling the assignment as part of the process - for kind in ("lesson",): # ("exam", "lesson"): + for kind in ("exam", "lesson"): for disable_assignment in (self.deactivate, self.unassign): # Create on Laptop A, single-user sync to tablet, disable, repeat diff --git a/kolibri/core/exams/kolibri_plugin.py b/kolibri/core/exams/kolibri_plugin.py new file mode 100644 index 00000000000..68da85ffefe --- /dev/null +++ b/kolibri/core/exams/kolibri_plugin.py @@ -0,0 +1,35 @@ +from .single_user_assignment_utils import ( + update_assignments_from_individual_syncable_exams, +) +from .single_user_assignment_utils import ( + update_individual_syncable_exams_from_assignments, +) +from kolibri.core.auth.hooks import FacilityDataSyncHook +from kolibri.plugins.hooks import register_hook + + +@register_hook +class SingleUserLessonSyncHook(FacilityDataSyncHook): + def pre_transfer( + self, + dataset_id, + local_is_single_user, + remote_is_single_user, + single_user_id, + context, + ): + # if we're about to send data to a single-user device, prep the syncable exam assignments + if context.is_producer and remote_is_single_user: + update_individual_syncable_exams_from_assignments(single_user_id) + + def post_transfer( + self, + dataset_id, + local_is_single_user, + remote_is_single_user, + single_user_id, + context, + ): + # if we've just received data on a single-user device, update the exams and assignments + if context.is_receiver and local_is_single_user: + update_assignments_from_individual_syncable_exams(single_user_id) diff --git a/kolibri/core/exams/migrations/0005_individualsyncableexam.py b/kolibri/core/exams/migrations/0005_individualsyncableexam.py new file mode 100644 index 00000000000..e3d2cb15148 --- /dev/null +++ b/kolibri/core/exams/migrations/0005_individualsyncableexam.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.29 on 2021-07-27 03:17 +from __future__ import unicode_literals + +import django.db.models.deletion +import morango.models.fields.uuids +from django.conf import settings +from django.db import migrations +from django.db import models + +import kolibri.core.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("kolibriauth", "0019_collection_no_mptt"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("exams", "0004_exam_add_dates_opened_created_and_archived"), + ] + + operations = [ + migrations.CreateModel( + name="IndividualSyncableExam", + fields=[ + ( + "id", + morango.models.fields.uuids.UUIDField( + editable=False, primary_key=True, serialize=False + ), + ), + ( + "_morango_dirty_bit", + models.BooleanField(default=True, editable=False), + ), + ("_morango_source_id", models.CharField(editable=False, max_length=96)), + ( + "_morango_partition", + models.CharField(editable=False, max_length=128), + ), + ("exam_id", models.UUIDField()), + ("serialized_exam", kolibri.core.fields.JSONField()), + ( + "collection", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="kolibriauth.Collection", + ), + ), + ( + "dataset", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="kolibriauth.FacilityDataset", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/kolibri/core/exams/models.py b/kolibri/core/exams/models.py index 56eb55d25b8..6071a61c980 100644 --- a/kolibri/core/exams/models.py +++ b/kolibri/core/exams/models.py @@ -186,3 +186,53 @@ def calculate_source_id(self): def calculate_partition(self): return self.dataset_id + + +class IndividualSyncableExam(AbstractFacilityDataModel): + """ + Represents a Exam and its assignment to a particular user + in such a way that it can be synced to a single-user device. + Note: This is not the canonical representation of a user's + relation to an exam (which is captured in an ExamAssignment + combined with a user's Membership in an associated Collection; + the purpose of this model is as a derived/denormalized + representation of a specific user's exam assignments). + """ + + morango_model_name = "individualsyncableexam" + + user = models.ForeignKey(FacilityUser) + collection = models.ForeignKey(Collection) + exam_id = models.UUIDField() + + serialized_exam = JSONField() + + def infer_dataset(self, *args, **kwargs): + return self.cached_related_dataset_lookup("user") + + def calculate_source_id(self): + return self.exam_id + + def calculate_partition(self): + return "{dataset_id}:user-ro:{user_id}".format( + dataset_id=self.dataset_id, user_id=self.user_id + ) + + @classmethod + def serialize_exam(cls, exam): + serialized = exam.serialize() + for key in [ + "active", + "creator_id", + "date_created", + "date_activated", + "collection_id", + ]: + serialized.pop(key, None) + return serialized + + @classmethod + def deserialize_exam(cls, serialized_exam): + exam = Exam.deserialize(serialized_exam) + exam.active = True + return exam diff --git a/kolibri/core/exams/single_user_assignment_utils.py b/kolibri/core/exams/single_user_assignment_utils.py new file mode 100644 index 00000000000..bff21021d3e --- /dev/null +++ b/kolibri/core/exams/single_user_assignment_utils.py @@ -0,0 +1,106 @@ +from .models import ExamAssignment +from .models import IndividualSyncableExam +from kolibri.core.auth.management.utils import DisablePostDeleteSignal + + +def update_individual_syncable_exams_from_assignments(user_id): + """ + Updates the set of IndividualSyncableExam objects for the user. + """ + syncableexams = IndividualSyncableExam.objects.filter(user_id=user_id) + assignments = ExamAssignment.objects.filter( + collection__membership__user_id=user_id, exam__active=True + ).distinct() + + # get a list of all active assignments that don't have a syncable exam + to_create = assignments.exclude(exam_id__in=syncableexams.values_list("exam_id")) + + # get a list of all syncable exams that still have an active assignment + to_update = syncableexams.filter(exam_id__in=assignments.values_list("exam_id")) + + # get a list of all syncable exams that don't have an active assignment anymore + to_delete = syncableexams.exclude(exam_id__in=assignments.values_list("exam_id")) + + # create new syncable exam objects for all new assignments + for assignment in to_create: + IndividualSyncableExam.objects.create( + user_id=user_id, + exam_id=assignment.exam_id, + serialized_exam=IndividualSyncableExam.serialize_exam(assignment.exam), + collection=assignment.collection, + ) + + # update existing syncable exam objects for all active assignments + for syncableexam in to_update: + assignment = assignments.get(exam_id=syncableexam.exam_id) + updated_serialization = IndividualSyncableExam.serialize_exam(assignment.exam) + if ( + syncableexam.serialized_exam != updated_serialization + or syncableexam.collection_id != assignment.collection_id + ): + syncableexam.serialized_exam = updated_serialization + syncableexam.collection_id = assignment.collection_id + syncableexam.save() + + # delete syncable exam objects that don't have an active assignment anymore + to_delete.delete() + + +def update_assignments_from_individual_syncable_exams(user_id): + """ + Looks at IndividualSyncableExams for a user and creates/deletes + the corresponding Exams and ExamAssignments as needed. + """ + syncableexams = IndividualSyncableExam.objects.filter(user_id=user_id) + assignments = ExamAssignment.objects.filter( + collection__membership__user_id=user_id, exam__active=True + ).distinct() + + # get a list of all syncable exams that aren't locally assigned + to_create = syncableexams.exclude(exam_id__in=assignments.values_list("exam_id")) + + # get a list of all assignments that may need updating from syncable exams + to_update = assignments.filter(exam_id__in=syncableexams.values_list("exam_id")) + + # get a list of all active assignments that no longer have a syncable exam + to_delete = assignments.exclude(exam_id__in=syncableexams.values_list("exam_id")) + + # create new assignments and exams for all new syncable exam objects + for syncableexam in to_create: + + exam = IndividualSyncableExam.deserialize_exam(syncableexam.serialized_exam) + exam.collection = syncableexam.collection + # shouldn't need to set this field (as it's nullable, according to the model definition, but got errors) + exam.creator_id = user_id + exam.save(update_dirty_bit_to=None) + + try: + ExamAssignment.objects.get( + collection=syncableexam.collection, + exam=exam, + ) + except ExamAssignment.DoesNotExist: + assignment = ExamAssignment( + collection=syncableexam.collection, + exam=exam, + assigned_by_id=user_id, # failed validation without this, so pretend it's self-assigned + ) + assignment.save(update_dirty_bit_to=None) + + # update existing exam/assignment objects for all syncable exams + for assignment in to_update: + syncableexam = syncableexams.get(exam_id=assignment.exam_id) + updated_serialization = IndividualSyncableExam.serialize_exam(assignment.exam) + if ( + syncableexam.serialized_exam != updated_serialization + or syncableexam.collection_id != assignment.collection_id + ): + exam = IndividualSyncableExam.deserialize_exam(syncableexam.serialized_exam) + exam.save(update_dirty_bit_to=None) + assignment.exam = exam + assignment.collection_id = syncableexam.collection_id + assignment.save(update_dirty_bit_to=None) + + # delete exams/assignments that no longer have a syncable exam object + with DisablePostDeleteSignal(): + to_delete.delete() diff --git a/kolibri/core/lessons/kolibri_plugin.py b/kolibri/core/lessons/kolibri_plugin.py new file mode 100644 index 00000000000..b5a2915b4e6 --- /dev/null +++ b/kolibri/core/lessons/kolibri_plugin.py @@ -0,0 +1,35 @@ +from .single_user_assignment_utils import ( + update_assignments_from_individual_syncable_lessons, +) +from .single_user_assignment_utils import ( + update_individual_syncable_lessons_from_assignments, +) +from kolibri.core.auth.hooks import FacilityDataSyncHook +from kolibri.plugins.hooks import register_hook + + +@register_hook +class SingleUserLessonSyncHook(FacilityDataSyncHook): + def pre_transfer( + self, + dataset_id, + local_is_single_user, + remote_is_single_user, + single_user_id, + context, + ): + # if we're about to send data to a single-user device, prep the syncable lesson assignments + if context.is_producer and remote_is_single_user: + update_individual_syncable_lessons_from_assignments(single_user_id) + + def post_transfer( + self, + dataset_id, + local_is_single_user, + remote_is_single_user, + single_user_id, + context, + ): + # if we've just received data on a single-user device, update the lessons and assignments + if context.is_receiver and local_is_single_user: + update_assignments_from_individual_syncable_lessons(single_user_id) diff --git a/kolibri/core/lessons/single_user_assignment_utils.py b/kolibri/core/lessons/single_user_assignment_utils.py index 923f89d1e41..5ff1dbf6042 100644 --- a/kolibri/core/lessons/single_user_assignment_utils.py +++ b/kolibri/core/lessons/single_user_assignment_utils.py @@ -1,81 +1,15 @@ -import json - -from morango.sync.context import LocalSessionContext - from .models import IndividualSyncableLesson from .models import LessonAssignment -from kolibri.core.auth.constants.morango_sync import ScopeDefinitions from kolibri.core.auth.management.utils import DisablePostDeleteSignal -from kolibri.core.auth.models import FacilityUser - - -def _get_our_cert(context): - ss = context.sync_session - return ss.server_certificate if ss.is_server else ss.client_certificate - - -def _get_their_cert(context): - ss = context.sync_session - return ss.client_certificate if ss.is_server else ss.server_certificate - - -def _this_side_using_single_user_cert(context): - return _get_our_cert(context).scope_definition_id == ScopeDefinitions.SINGLE_USER - - -def _other_side_using_single_user_cert(context): - return _get_their_cert(context).scope_definition_id == ScopeDefinitions.SINGLE_USER - - -def _get_user_for_single_user_sync(context): - cert = None - if _other_side_using_single_user_cert(context): - cert = _get_their_cert(context) - elif _this_side_using_single_user_cert(context): - cert = _get_our_cert(context) - else: - return "" - return FacilityUser.objects.get(id=json.loads(cert.scope_params)["user_id"]) - - -def _initializing_handler(context): - assert context is not None - - if ( - isinstance(context, LocalSessionContext) - and context.is_producer - and _other_side_using_single_user_cert(context) - ): - update_individual_syncable_lessons_from_assignments( - _get_user_for_single_user_sync(context) - ) - - -def _cleanup_handler(context): - assert context is not None - - if ( - isinstance(context, LocalSessionContext) - and context.is_receiver - and _this_side_using_single_user_cert(context) - ): - update_assignments_from_individual_syncable_lessons( - _get_user_for_single_user_sync(context) - ) - - -def register_single_user_sync_lesson_handlers(session_controller): - session_controller.signals.initializing.completed.connect(_initializing_handler) - session_controller.signals.cleanup.completed.connect(_cleanup_handler) -def update_individual_syncable_lessons_from_assignments(user): +def update_individual_syncable_lessons_from_assignments(user_id): """ Updates the set of IndividualSyncableLesson objects for the user. """ - syncablelessons = IndividualSyncableLesson.objects.filter(user=user) + syncablelessons = IndividualSyncableLesson.objects.filter(user_id=user_id) assignments = LessonAssignment.objects.filter( - collection__membership__user=user, lesson__is_active=True + collection__membership__user_id=user_id, lesson__is_active=True ).distinct() # get a list of all active assignments that don't have a syncable lesson @@ -96,7 +30,7 @@ def update_individual_syncable_lessons_from_assignments(user): # create new syncable lesson objects for all new assignments for assignment in to_create: IndividualSyncableLesson.objects.create( - user=user, + user_id=user_id, lesson_id=assignment.lesson_id, serialized_lesson=IndividualSyncableLesson.serialize_lesson( assignment.lesson @@ -122,14 +56,14 @@ def update_individual_syncable_lessons_from_assignments(user): to_delete.delete() -def update_assignments_from_individual_syncable_lessons(user): +def update_assignments_from_individual_syncable_lessons(user_id): """ Looks at IndividualSyncableLessons for a user and creates/deletes the corresponding Lessons and LessonAssignments as needed. """ - syncablelessons = IndividualSyncableLesson.objects.filter(user=user) + syncablelessons = IndividualSyncableLesson.objects.filter(user_id=user_id) assignments = LessonAssignment.objects.filter( - collection__membership__user=user, lesson__is_active=True + collection__membership__user_id=user_id, lesson__is_active=True ).distinct() # get a list of all syncable lessons that aren't locally assigned @@ -155,7 +89,7 @@ def update_assignments_from_individual_syncable_lessons(user): ) lesson.collection = syncablelesson.collection # shouldn't need to set this field (as it's nullable, according to the model definition, but got errors) - lesson.created_by = user + lesson.created_by_id = user_id lesson.save(update_dirty_bit_to=None) try: @@ -167,7 +101,7 @@ def update_assignments_from_individual_syncable_lessons(user): assignment = LessonAssignment( collection=syncablelesson.collection, lesson=lesson, - assigned_by=user, # failed validation without this, so pretend it's self-assigned + assigned_by_id=user_id, # failed validation without this, so pretend it's self-assigned ) assignment.save(update_dirty_bit_to=None) diff --git a/kolibri/core/lessons/viewsets.py b/kolibri/core/lessons/viewsets.py index 59efad7d207..c2a849064c2 100644 --- a/kolibri/core/lessons/viewsets.py +++ b/kolibri/core/lessons/viewsets.py @@ -1,5 +1,4 @@ from django_filters.rest_framework import DjangoFilterBackend -from morango.api.viewsets import session_controller from .serializers import LessonSerializer from kolibri.core.api import ValuesViewset @@ -8,13 +7,8 @@ from kolibri.core.auth.constants.collection_kinds import ADHOCLEARNERSGROUP from kolibri.core.lessons.models import Lesson from kolibri.core.lessons.models import LessonAssignment -from kolibri.core.lessons.single_user_assignment_utils import ( - register_single_user_sync_lesson_handlers, -) from kolibri.core.query import annotate_array_aggregate -register_single_user_sync_lesson_handlers(session_controller) - def _ensure_raw_dict(d): if hasattr(d, "dict"): diff --git a/requirements/base.txt b/requirements/base.txt index f3a887763f5..53ef13eaafe 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -16,7 +16,7 @@ le-utils==0.1.24 kolibri_exercise_perseus_plugin==1.3.5 jsonfield==2.0.2 requests-toolbelt==0.8.0 -morango==0.6.1 +morango==0.6.2 tzlocal==2.1 pytz==2020.5 python-dateutil==2.7.5