Merge 0.14.7 to develop

.editorconfig

@@ -14,3 +14,6 @@ indent_size = 4
 
 [Makefile]
 indent_style = tab
+
+[*.feature]
+indent_style = tab

CHANGELOG.md

@@ -2,13 +2,35 @@
 
 List of the most important changes for each release.
 
+## 0.14.7
+
+### Internationalization and localization
+
+- Updated localizations
+
+### Fixed
+- #7766 Content imported by administrators was not immediately available for learners to use
+- #7869 Unlisted channels would not appear in list in channel import-workflow after providing token
+- #7810 Learners' new passwords were not being validated on the Sign-In page
+- #7764 Users' progress on resources was not being properly logged, making it difficult to complete them
+- #8003, #8004, #8010 Sign-ins could cause the server to crash if database was locked
+- #8003, #7947 Issues downloading CSV files on Windows
+
+### Changed
+
+- #7735 Filtering on lists of users returns ranked and approximate matches
+- #7733 Resetting a facility's settings respects the preset (e.g. formal, informal, nonformal) chosen for it during setup
+- #7823 Improved performance on coach pages for facilities with large numbers of classrooms and groups
+
+([0.14.7 Github milestone](https://github.com/learningequality/kolibri/issues?q=label%3Achangelog+milestone%3A0.14.7))
+
 ## 0.14.6
 
 ### Fixed
 
-* #7725 On Firefox, text in Khmer, Hindi, Marathi, and other languages did not render properly.
-* #7722, #7488 After viewing a restricted page, then signing in, users were not redirected back to the restricted page.
-* #7597, #7612 Quiz creation workflow did not properly validate the number of questions
+- #7725 On Firefox, text in Khmer, Hindi, Marathi, and other languages did not render properly.
+- #7722, #7488 After viewing a restricted page, then signing in, users were not redirected back to the restricted page.
+- #7597, #7612 Quiz creation workflow did not properly validate the number of questions
 
 ([0.14.6 Github milestone](https://github.com/learningequality/kolibri/issues?q=label%3Achangelog+milestone%3A0.14.6))
 
@@ -18,21 +40,21 @@ List of the most important changes for each release.
 
 ### Changed
 
-* File downloads now run concurrently, taking better advantage of a device's bandwidth and reducing the time needed to import resources from Kolibri Studio or other content sources
-* When setting up a new device using the [Setup Wizard's "Quick Start" option](https://kolibri.readthedocs.io/en/latest/install/initial_setup.html#quick-start), the ["Allow learners to create accounts" setting](https://kolibri.readthedocs.io/en/latest/install/initial_setup.html#quick-start) is enabled by default.
-* The `provisiondevice` management command no longer converts the user-provided facility name to all lower-case
-* Markdown descriptions for resources now preserve line breaks from the original source
+- File downloads now run concurrently, taking better advantage of a device's bandwidth and reducing the time needed to import resources from Kolibri Studio or other content sources
+- When setting up a new device using the [Setup Wizard's "Quick Start" option](https://kolibri.readthedocs.io/en/latest/install/initial_setup.html#quick-start), the ["Allow learners to create accounts" setting](https://kolibri.readthedocs.io/en/latest/install/initial_setup.html#quick-start) is enabled by default.
+- The `provisiondevice` management command no longer converts the user-provided facility name to all lower-case
+- Markdown descriptions for resources now preserve line breaks from the original source
 
 ### Fixed
 
-* Multiple bugs when creating, editing, and copying quizzes/lessons
-* Multiple bugs when navigating throughout the Coach page
-* Multiple bugs specific to Kolibri servers using PostgreSQL
-* On Safari, sections of the Facility > Data page would disappear unexpectedly after syncing a facility
-* On IE11, it was not possible to setup a new device by importing a facility
-* Missing thumbnails on resource cards when searching/browsing in channels
-* Numerous visual and accessibility issues
-* Facilities could not be renamed if the only changes were to the casing of the name (e.g. changing "Facility" to "FACILITY")
+- Multiple bugs when creating, editing, and copying quizzes/lessons
+- Multiple bugs when navigating throughout the Coach page
+- Multiple bugs specific to Kolibri servers using PostgreSQL
+- On Safari, sections of the Facility > Data page would disappear unexpectedly after syncing a facility
+- On IE11, it was not possible to setup a new device by importing a facility
+- Missing thumbnails on resource cards when searching/browsing in channels
+- Numerous visual and accessibility issues
+- Facilities could not be renamed if the only changes were to the casing of the name (e.g. changing "Facility" to "FACILITY")
 
 ([0.14.5 Github milestone](https://github.com/learningequality/kolibri/issues?q=label%3Achangelog+milestone%3A0.14.5))
 

docker/build_whl.dockerfile

@@ -39,7 +39,6 @@ WORKDIR /kolibri
 # Python dependencies
 COPY requirements/ requirements/
 RUN echo '--- Installing Python dependencies' && \
-    pip install -r requirements/dev.txt && \
     pip install -r requirements/build.txt
 
 # Set yarn cache folder for easy binding during runtime

kolibri/core/analytics/utils.py

@@ -45,8 +45,8 @@
 from kolibri.core.logger.models import ExamLog
 from kolibri.core.logger.models import UserSessionLog
 from kolibri.core.tasks.main import scheduler
-from kolibri.core.tasks.utils import db_task_write_lock
 from kolibri.core.tasks.utils import get_current_job
+from kolibri.core.utils.lock import db_lock
 from kolibri.utils import conf
 from kolibri.utils.server import installation_type
 from kolibri.utils.time_utils import local_now
@@ -392,7 +392,7 @@ def extract_channel_statistics(channel):
 def create_and_update_notifications(data, source):
     messages = [obj for obj in data.get("messages", []) if obj.get("msg_id")]
     excluded_ids = [obj.get("msg_id") for obj in messages]
-    with db_task_write_lock:
+    with db_lock():
         PingbackNotification.objects.filter(source=source).exclude(
             id__in=excluded_ids
         ).update(active=False)
@@ -407,7 +407,7 @@ def create_and_update_notifications(data, source):
             "source": source,
             "active": True,
         }
-        with db_task_write_lock:
+        with db_lock():
             PingbackNotification.objects.update_or_create(
                 id=new_msg["id"], defaults=new_msg
             )

kolibri/core/auth/csv_utils.py

@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
 import csv
-import io
 import logging
 import os
-import sys
 from collections import OrderedDict
 from functools import partial
 
@@ -18,6 +16,7 @@
 from kolibri.core.auth.models import Facility
 from kolibri.core.auth.models import FacilityUser
 from kolibri.core.query import SQCount
+from kolibri.core.utils.csv import open_csv_for_writing
 
 
 logger = logging.getLogger(__name__)
@@ -186,10 +185,7 @@ def csv_file_generator(facility, filepath, overwrite=True, demographic=False):
         column for column in db_columns if demographic or column not in DEMO_FIELDS
     )
 
-    if sys.version_info[0] < 3:
-        csv_file = io.open(filepath, "wb")
-    else:
-        csv_file = io.open(filepath, "w", newline="")
+    csv_file = open_csv_for_writing(filepath)
 
     with csv_file as f:
         writer = csv.DictWriter(f, header_labels)

kolibri/core/auth/management/commands/bulkexportusers.py

@@ -1,9 +1,7 @@
 import csv
-import io
 import logging
 import ntpath
 import os
-import sys
 from collections import OrderedDict
 from functools import partial
 from tempfile import mkstemp
@@ -28,6 +26,7 @@
 from kolibri.core.query import GroupConcatSubquery
 from kolibri.core.tasks.management.commands.base import AsyncCommand
 from kolibri.core.tasks.utils import get_current_job
+from kolibri.core.utils.csv import open_csv_for_writing
 from kolibri.utils import conf
 
 try:
@@ -161,10 +160,7 @@ def csv_file_generator(facility, filepath, overwrite=True):
 
     header_labels = labels.values()
 
-    if sys.version_info[0] < 3:
-        csv_file = io.open(filepath, "wb")
-    else:
-        csv_file = io.open(filepath, "w", newline="")
+    csv_file = open_csv_for_writing(filepath)
 
     with csv_file as f:
         writer = csv.DictWriter(f, header_labels)

kolibri/core/auth/management/commands/bulkimportusers.py

@@ -23,6 +23,7 @@
 from kolibri.core.auth.models import Membership
 from kolibri.core.tasks.management.commands.base import AsyncCommand
 from kolibri.core.tasks.utils import get_current_job
+from kolibri.core.utils.csv import open_csv_for_reading
 
 try:
     FileNotFoundError
@@ -465,8 +466,8 @@ def csv_values_validation(self, reader, header_translation):
         )
 
     def csv_headers_validation(self, filepath):
-        # open using default OS encoding
-        with open(filepath) as f:
+        csv_file = open_csv_for_reading(filepath)
+        with csv_file as f:
             header = next(csv.reader(f, strict=True))
             has_header = False
             self.header_translation = {
@@ -852,7 +853,8 @@ def handle_async(self, *args, **options):
             self.exit_if_error()
             self.progress_update(1)  # state=csv_headers
             try:
-                with open(filepath) as f:
+                csv_file = open_csv_for_reading(filepath)
+                with csv_file as f:
                     reader = csv.DictReader(f, strict=True)
                     per_line_errors, classes, users, roles = self.csv_values_validation(
                         reader, self.header_translation

kolibri/core/auth/management/commands/importusers.py

@@ -15,6 +15,7 @@
 from kolibri.core.auth.models import Classroom
 from kolibri.core.auth.models import Facility
 from kolibri.core.auth.models import FacilityUser
+from kolibri.core.utils.csv import open_csv_for_reading
 
 logger = logging.getLogger(__name__)
 
@@ -196,8 +197,8 @@ def handle(self, *args, **options):
 
         fieldnames = input_fields + tuple(val for val in labels.values())
 
-        # open using default OS encoding
-        with open(options["filepath"]) as f:
+        csv_file = open_csv_for_reading(options["filepath"])
+        with csv_file as f:
             header = next(csv.reader(f, strict=True))
             has_header = False
             if all(col in fieldnames for col in header):
@@ -212,8 +213,8 @@ def handle(self, *args, **options):
                     "Mix of valid and invalid header labels found in first row"
                 )
 
-        # open using default OS encoding
-        with open(options["filepath"]) as f:
+        csv_file = open_csv_for_reading(options["filepath"])
+        with csv_file as f:
             if has_header:
                 reader = csv.DictReader(f, strict=True)
             else:

kolibri/core/auth/management/commands/sync.py

@@ -23,7 +23,7 @@
 from kolibri.core.logger.utils.data import bytes_for_humans
 from kolibri.core.tasks.exceptions import UserCancelledError
 from kolibri.core.tasks.management.commands.base import AsyncCommand
-from kolibri.core.tasks.utils import db_task_write_lock
+from kolibri.core.utils.lock import db_lock
 from kolibri.utils import conf
 
 DATA_PORTAL_SYNCING_BASE_URL = conf.OPTIONS["Urls"]["DATA_PORTAL_SYNCING_BASE_URL"]
@@ -108,6 +108,7 @@ def handle_async(self, *args, **options):  # noqa C901
             call_command("loaddata", "scopedefinitions")
 
         dataset_cache.clear()
+        dataset_cache.activate()
 
         # try to connect to server
         controller = MorangoProfileController(PROFILE_FACILITY_DATA)
@@ -204,6 +205,7 @@ def handle_async(self, *args, **options):  # noqa C901
             self.job.extra_metadata.update(sync_state=State.COMPLETED)
             self.job.save_meta()
 
+        dataset_cache.deactivate()
         logger.info("Syncing has been completed.")
 
     @contextmanager
@@ -214,7 +216,7 @@ def _lock(self):
             cancellable = self.job.cancellable
             self.job.save_as_cancellable(cancellable=False)
 
-        with db_task_write_lock:
+        with db_lock():
             yield
 
         if self.job:

kolibri/core/auth/models.py

@@ -23,13 +23,13 @@
 from __future__ import unicode_literals
 
 import logging
+from threading import local
 
 import six
 from django.contrib.auth.models import AbstractBaseUser
 from django.contrib.auth.models import AnonymousUser
 from django.contrib.auth.models import UserManager
 from django.core import validators
-from django.core.cache import cache
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.exceptions import ValidationError
 from django.db import models
@@ -76,11 +76,37 @@
 from kolibri.core.device.utils import set_device_settings
 from kolibri.core.errors import KolibriValidationError
 from kolibri.core.fields import DateTimeTzField
-from kolibri.core.utils.cache import NamespacedCacheProxy
 from kolibri.utils.time_utils import local_now
 
 logger = logging.getLogger(__name__)
-dataset_cache = NamespacedCacheProxy(cache, "dataset")
+
+
+class DatasetCache(local):
+    def __init__(self):
+        self.deactivate()
+
+    def activate(self):
+        self._active = True
+
+    def deactivate(self):
+        self._active = False
+        self.clear()
+
+    def clear(self):
+        self._cache = {}
+
+    def get(self, key):
+        if self._active:
+            return self._cache.get(key)
+        return None
+
+    def set(self, key, dataset_id):
+        if self._active:
+            self._cache[key] = dataset_id
+        return None
+
+
+dataset_cache = DatasetCache()
 
 
 def _has_permissions_class(obj):
@@ -218,7 +244,7 @@ def cached_related_dataset_lookup(self, related_obj_name):
                 dataset_id = getattr(self, related_obj_name).dataset_id
             except ObjectDoesNotExist as e:
                 raise ValidationError(e)
-            dataset_cache.set(key, dataset_id, 60 * 10)
+            dataset_cache.set(key, dataset_id)
         return dataset_id
 
     def calculate_source_id(self):

kolibri/core/auth/test/test_bulk_import.py

@@ -1,5 +1,4 @@
 import csv
-import io
 import sys
 import tempfile
 from uuid import uuid4
@@ -15,6 +14,7 @@
 from kolibri.core.auth.constants import role_kinds
 from kolibri.core.auth.models import Classroom
 from kolibri.core.auth.models import FacilityUser
+from kolibri.core.utils.csv import open_csv_for_writing
 
 if sys.version_info[0] < 3:
     from cStringIO import StringIO
@@ -118,10 +118,7 @@ def setUp(self):
     def create_csv(self, filepath, rows, remove_uuid=False):
         header_labels = list(labels.values())
 
-        if sys.version_info[0] < 3:
-            csv_file = io.open(filepath, "wb")
-        else:
-            csv_file = io.open(filepath, "w", newline="")
+        csv_file = open_csv_for_writing(filepath)
 
         with csv_file as f:
             writer = csv.writer(f)