Merge pull request #5584 from learningequality/0.17.x

0.17.x into master (0.17.5 final)

.buildkite/build_and_upload_artifact.sh

@@ -4,7 +4,6 @@ set -euo pipefail
 
 make dockerenvdist
 buildkite-agent artifact upload 'dist/*.whl'
-buildkite-agent artifact upload 'dist/*.zip'
 buildkite-agent artifact upload 'dist/*.tar.gz'
 buildkite-agent artifact upload 'dist/*.pex'
 buildkite-agent artifact upload 'dist/*.exe'

.buildkite/setup_and_upload_artifacts.sh

@@ -19,7 +19,6 @@ mkdir -p installer
 echo "Downloading artifacts..."
 buildkite-agent artifact download 'dist/*.pex' dist/
 buildkite-agent artifact download 'dist/*.whl' dist/
-buildkite-agent artifact download 'dist/*.zip' dist/
 buildkite-agent artifact download 'dist/*.tar.gz' dist/
 buildkite-agent artifact download 'installer/*.exe' installer/
 

Dockerfile

@@ -1,11 +1,10 @@
 FROM ubuntu:xenial
 
 # install latest python and nodejs
-RUN apt-get -y update
-RUN apt-get install -y software-properties-common curl
-RUN add-apt-repository ppa:voronov84/andreyv
-RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -
-RUN apt-get -y update && apt-get install -y python2.7 python-pip git nodejs gettext wget
+RUN apt-get -y update && \
+  apt-get install -y software-properties-common curl && \
+  curl -sL https://deb.nodesource.com/setup_6.x | bash - && \
+  apt-get -y update && apt-get install -y python2.7 python-pip git nodejs gettext wget
 
 COPY . /kalite
 VOLUME /kalitedist/

Makefile

@@ -97,7 +97,7 @@ assets:
 	rm -rf kalite/database/templates/
 	mkdir -p kalite/database/templates/
 	cp .kalite_dist_tmp/database/data.sqlite kalite/database/templates/
-	bin/kalite manage retrievecontentpack empty en --foreground --template
+	bin/kalite manage retrievecontentpack empty en --template
 
 msgids:
 	export IGNORE_PATTERNS="*/kalite/static-libraries/*,*/LC_MESSAGES/*,*/kalite/packages/dist/*,*/kalite/packages/bundled/django/*,*/kalite/*/bundles/bundle*.js,*/kalite/*/js/i18n/*.js" ;\
@@ -123,6 +123,9 @@ dist: clean docs assets
 	make clean-pyc
 	python setup.py sdist --formats=$(format)
 	python setup.py bdist_wheel
+	pip install -t kalite/packages/dist -r "requirements.txt"
+	rm -rf kalite/packages/dist/*.dist-info  # pip installs from PyPI will complain if we have more than one dist-info directory.
+	rm -r kalite/packages/dist/man kolibri/dist/bin || true # remove the two folders introduced by pip 10
 	python setup.py sdist --formats=$(format) --static
 	python setup.py bdist_wheel --static  --no-clean
 	ls -l dist

docs/index.rst

@@ -46,6 +46,15 @@ Khan Academy content (videos and exercises) from a local server, with
 points and progress-tracking, without needing internet connectivity.
 
 
+Translations
+------------
+
+KA Lite is released in many languages, each with its own subset of Khan Academy contents translated: Polish, Georgian, German, Spanish, Xhosa, Indonesian, Burmese, Swahili, Tamil, Hindi, Portuguese (Brazilian and Portuguese), Bulgarian, Danish, Arabic, Bengali, Kannada, and French. See :ref:`adding_languages`.
+
+Documentation: This documentation is also available
+`in French (PDF) <http://pantry.learningequality.org/downloads/ka-lite/0.17/docs_fr.pdf>`__.
+
+
 Get involved!
 -------------
 

docs/installguide/advanced.rst

@@ -62,7 +62,7 @@ and if you are connected to the internet, this will also give you automatic upda
 To add the PPA as a repository on an apt-based system, you need to ensure that a few libraries are present, and then add our repository and the public key that packages are signed with::
 
 
-    sudo apt-get install software-properties-common python-software-properties
+    sudo apt-get install dirmngr
     sudo su -c 'echo "deb http://ppa.launchpad.net/learningequality/ka-lite/ubuntu xenial main" > /etc/apt/sources.list.d/ka-lite.list'
     sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 74F88ADB3194DD81
     sudo apt-get update

docs/installguide/release_notes.rst

@@ -9,13 +9,24 @@ to read the release notes.
   ``0.15.x`` to ``0.17.x`` is not guaranteed to work.
 
 
+0.17.5
+------
+
+Bug fixes
+^^^^^^^^^
+
+* Slow download using ``retrievecontentpack`` on Raspberry Pi :url-issue:`5575`
+* Customized content packs may fail depending on the Zip program used :url-issue:`5476`
+
+
 0.17.4
 ------
 
 Added
 ^^^^^
 
 * Progress displayed while downloading content packs :url-issue:`5356`
+* Customizable welcome message setting ``KALITE_WELCOME_MESSAGE`` displayed to admin users on first login :url-issue:`5522`
 
 Bug fixes
 ^^^^^^^^^
@@ -27,10 +38,6 @@ Bug fixes
 * Also delete content database when deleting a content pack :url-issue:`5545`
 * Simplified login is now working when there are 1,000 or more users registered in a facility. :url-issue:`5523`
 
-New Features
-^^^^^^^^^^^^
-
-* Customizable welcome message setting ``KALITE_WELCOME_MESSAGE`` displayed to admin users on first login :url-issue:`5522`
 
 Developers
 ^^^^^^^^^^
@@ -41,6 +48,13 @@ Developers
 Known issues
 ^^^^^^^^^^^^
 
+* The English **Math** topic (top-level) contains a number of misplaced
+  subtopics in the 0.17 series. The problem has propagated from upstream data.
+  The misplacement clutters/obstructs the navigation for students, and it has
+  been a priority to fix this in 0.18, released shortly after 0.17.4.
+  But it's recommended to upgrade to 0.18 and download new video contents
+  following from the new content pack (~450 videos). See below for tips on how
+  to install a content pack from a different KA Lite release. :url-issue:`5561`
 * It isn't possible to cancel video downloads if a video is downloading while
   the connection is switched off.
 * **Chrome 55-56** has issues scrolling the menus on touch devices. Upgrading to Chrome 57 fixes this. :url-issue:`5407`
@@ -51,6 +65,17 @@ Known issues
 * A limited number of exercises with radio buttons have problems displaying :url-issue:`5172`
 
 
+.. tip::
+  It is possible to install a 0.16 content pack in a 0.17 release as a
+  work-around. Download a specific content pack and then install it from
+  command-line. Example::
+    
+    # Download content pack to "en_0.16.zip"
+    $ wget -r -O en_0.16.zip http://pantry.learningequality.org/downloads/ka-lite/0.16/content/contentpacks/en.zip
+    # Install "en_0.16.zip"
+    $ kalite manage retrievecontentpack local en en_0.16.zip
+
+
 0.17.3
 ------
 

docs/usermanual/userman_admin.rst

@@ -495,6 +495,7 @@ After you copied in the new video files or changed the ``CONTENT_ROOT`` path, yo
 
 4. Once the scan is completed, video content will be ready for Learners to watch!
 
+.. _adding_languages:
 
 Adding Languages
 ----------------
@@ -675,6 +676,8 @@ _________________
   institutions where permissions should be reserved for admins.
 * ``USER_LOG_MAX_RECORDS_PER_USER = <max user log records per user> (default = 0 [disabled], -1=unlimited logs)``
   In order to keep local data in the ``UserLog`` model, detailing usage, you can choose the number of ``UserLog`` objects that you wish to retain. These objects are not sync'ed.
+* ``SIMPLIFIED_LOGIN = <True or False> (default = False)``.
+  Switches off passwords in the Learner's login modal.
 
 
 Online synchronization

kalite/distributed/management/commands/retrievecontentpack.py

@@ -23,6 +23,7 @@
 from peewee import SqliteDatabase
 from kalite.topic_tools.content_models import Item, AssessmentItem
 from requests.exceptions import ConnectionError, HTTPError
+from fle_utils.django_utils.command import LocaleAwareCommand
 
 
 logger = logging.getLogger(__name__)
@@ -41,7 +42,19 @@ class Command(UpdatesStaticCommand):
 
     """
 
-    option_list = UpdatesStaticCommand.option_list + (
+
+    option_list = LocaleAwareCommand.option_list + (
+        make_option(
+            "", "--background",
+            action="store_true",
+            dest="background",
+            default=False,
+            help=(
+                "Run the command with job scheduler and database-backed "
+                "progress. Used when you retrieve a content pack before "
+                "initializing KA Lite's db."
+            )
+        ),
         make_option(
             "", "--template",
             action="store_true",
@@ -147,6 +160,9 @@ def download(self, *args, **options):
 
         lang = args[1]
 
+        # This rather hacky callback will add progress to the database-backed
+        # progress monitoriing, which is terrible for performance on for
+        # instance Raspberry Pi devices.
         def download_callback(fraction):
             percent = int(fraction * 100)
             self.update_stage(
@@ -223,7 +239,10 @@ def extract_subtitles(zf, lang):
 
     ensure_dir(SUBTITLE_DEST_DIR)
 
-    subtitles = (s for s in zf.namelist() if SUBTITLE_ZIP_DIR in s)
+    def is_subtitle_file(s):
+        return SUBTITLE_ZIP_DIR in s and len(s) > len(SUBTITLE_ZIP_DIR)
+
+    subtitles = (s for s in zf.namelist() if is_subtitle_file(s))
 
     for subtitle in subtitles:
         # files inside zipfiles may come with leading directories in their

kalite/testing/testrunner.py

@@ -179,7 +179,7 @@ def build_suite(self, test_labels, extra_tests, **kwargs):
         browser.quit()
 
         if not database_exists() or os.path.getsize(database_path()) < 1024 * 1024:
-            call_command("retrievecontentpack", "empty", "en", force=True, foreground=True)
+            call_command("retrievecontentpack", "empty", "en", force=True)
             logging.info("Successfully setup content database")
         else:
             logging.info("Content database already exists")

kalite/updates/api_views.py

@@ -205,7 +205,7 @@ def start_languagepack_download(request):
     data = json.loads(request.raw_post_data)  # Django has some weird post processing into request.POST, so use .body
     lang_code = lcode_to_ietf(data['lang'])
 
-    call_command_async('retrievecontentpack', 'download', lang_code)
+    call_command_async('retrievecontentpack', 'download', lang_code, background=True)
 
     return JsonResponseMessageSuccess(_("Successfully started language pack download for %(lang_name)s.") % {"lang_name": get_language_name(lang_code)})
 

kalite/updates/management/utils.py

@@ -29,7 +29,11 @@ class UpdatesCommand(LocaleAwareCommand):
     """
     Abstract class for sharing code across Dynamic and Static versions
     """
+
     option_list = LocaleAwareCommand.option_list + (
+        # WARNING: Because of issues having foreground as a default option, we
+        # are overwriting this with a --background option in
+        # retrievecontentpack
         make_option(
             "", "--foreground",
             action="store_true",
@@ -39,7 +43,8 @@ class UpdatesCommand(LocaleAwareCommand):
                 "Run the command without job scheduler and  database-backed "
                 "progress. Used when you retrieve a content pack before "
                 "initializing KA Lite's db."
-            )),
+            )
+        ),
     )
 
     def __init__(self, process_name=None, *args, **kwargs):
@@ -53,9 +58,10 @@ def setup(self, options):
         # Should be set by command's handle() method since there's no clear
         # call structure defined. Respected by the methods that update stuff
         # in the database.
-        self.foreground = options["foreground"]
+        self.foreground = options.get("foreground", False)
+        self.background = options.get("background", True)
 
-        if not self.foreground:
+        if not self.foreground and self.background:
             self.progress_log = UpdateProgressLog.get_active_log(process_name=self.process_name)
         else:
             self.progress_log = None

kalite/version.py

@@ -3,7 +3,7 @@
 # Must also be of the form N.N.N for internal use, where N is a non-negative integer
 MAJOR_VERSION = "0"
 MINOR_VERSION = "17"
-PATCH_VERSION = "4"
+PATCH_VERSION = "5"
 VERSION = "%s.%s.%s" % (MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION)
 SHORTVERSION = "%s.%s" % (MAJOR_VERSION, MINOR_VERSION)
 

setup.py

@@ -207,59 +207,10 @@ def run(self):
 if STATIC_BUILD:
 
     sys.stderr.write(
-        "This is a static build... invoking pip to put static dependencies in "
-        "kalite/packages/dist/\n"
+        "This is a static build... we hope you invoked this build through the "
+        "'make dist' target, because this populates kalite/packages/dist"
     )
 
-    STATIC_DIST_PACKAGES_DOWNLOAD_CACHE = os.path.join(where_am_i, '.pip-downloads')
-    STATIC_DIST_PACKAGES_TEMP = os.path.join(where_am_i, '.pip-temp')
-
-    # Create directory where dynamically created dependencies are put
-    if not os.path.exists(STATIC_DIST_PACKAGES_DOWNLOAD_CACHE):
-        os.mkdir(STATIC_DIST_PACKAGES_DOWNLOAD_CACHE)
-
-    # Should remove the temporary directory always
-    if not NO_CLEAN and os.path.exists(STATIC_DIST_PACKAGES_TEMP):
-        print("Removing previous temporary sources for pip {}".format(STATIC_DIST_PACKAGES_TEMP))
-        shutil.rmtree(STATIC_DIST_PACKAGES_TEMP)
-
-    # Install from pip
-
-    # Code modified from this example:
-    # http://threebean.org/blog/2011/06/06/installing-from-pip-inside-python-or-a-simple-pip-api/
-    import pip.commands.install
-
-    # Ensure we get output from pip
-    enable_log_to_stdout('pip.commands.install')
-
-    def install_distributions(distributions):
-        command = pip.commands.install.InstallCommand()
-        opts, ___ = command.parser.parse_args([])
-        opts.target_dir = STATIC_DIST_PACKAGES
-        opts.build_dir = STATIC_DIST_PACKAGES_TEMP
-        opts.download_cache = STATIC_DIST_PACKAGES_DOWNLOAD_CACHE
-        opts.isolated = True
-        opts.ignore_installed = True
-        opts.compile = False
-        opts.ignore_dependencies = False
-        # This is deprecated and will disappear in Pip 10
-        opts.use_wheel = False
-        # The below is not an option, then we skip mimeparse
-        # opts.no_binary = ':all:'  # Do not use any binary files (whl)
-        opts.no_clean = NO_CLEAN
-        command.run(opts, distributions)
-
-    # Install requirements into kalite/packages/dist
-    if DIST_BUILDING_COMMAND:
-        install_distributions(RAW_REQUIREMENTS)
-
-        # Now remove Django because it's bundled. It gets installed as an egg
-        # so unfortunately, the path is a bit dependent on the specific
-        # version installed (we pinned it for reliability in requirements.txt)
-        shutil.rmtree(
-            os.path.join(STATIC_DIST_PACKAGES, "Django-1.5.12-py2.7.egg-info"),
-        )
-
 # It's not a build command with --static or it's not a build command at all
 else: