diff --git a/.buildkite/build_and_upload_artifact.sh b/.buildkite/build_and_upload_artifact.sh index 394058dff0..f83521a4c6 100755 --- a/.buildkite/build_and_upload_artifact.sh +++ b/.buildkite/build_and_upload_artifact.sh @@ -4,7 +4,6 @@ set -euo pipefail make dockerenvdist buildkite-agent artifact upload 'dist/*.whl' -buildkite-agent artifact upload 'dist/*.zip' buildkite-agent artifact upload 'dist/*.tar.gz' buildkite-agent artifact upload 'dist/*.pex' buildkite-agent artifact upload 'dist/*.exe' diff --git a/.buildkite/setup_and_upload_artifacts.sh b/.buildkite/setup_and_upload_artifacts.sh index 70f244786f..6f9b759799 100755 --- a/.buildkite/setup_and_upload_artifacts.sh +++ b/.buildkite/setup_and_upload_artifacts.sh @@ -19,7 +19,6 @@ mkdir -p installer echo "Downloading artifacts..." buildkite-agent artifact download 'dist/*.pex' dist/ buildkite-agent artifact download 'dist/*.whl' dist/ -buildkite-agent artifact download 'dist/*.zip' dist/ buildkite-agent artifact download 'dist/*.tar.gz' dist/ buildkite-agent artifact download 'installer/*.exe' installer/ diff --git a/Dockerfile b/Dockerfile index 0c7f985f66..353ce71416 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,10 @@ FROM ubuntu:xenial # install latest python and nodejs -RUN apt-get -y update -RUN apt-get install -y software-properties-common curl -RUN add-apt-repository ppa:voronov84/andreyv -RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - -RUN apt-get -y update && apt-get install -y python2.7 python-pip git nodejs gettext wget +RUN apt-get -y update && \ + apt-get install -y software-properties-common curl && \ + curl -sL https://deb.nodesource.com/setup_6.x | bash - && \ + apt-get -y update && apt-get install -y python2.7 python-pip git nodejs gettext wget COPY . /kalite VOLUME /kalitedist/ diff --git a/Makefile b/Makefile index 0e455ccb61..2d33ce9897 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ assets: rm -rf kalite/database/templates/ mkdir -p kalite/database/templates/ cp .kalite_dist_tmp/database/data.sqlite kalite/database/templates/ - bin/kalite manage retrievecontentpack empty en --foreground --template + bin/kalite manage retrievecontentpack empty en --template msgids: export IGNORE_PATTERNS="*/kalite/static-libraries/*,*/LC_MESSAGES/*,*/kalite/packages/dist/*,*/kalite/packages/bundled/django/*,*/kalite/*/bundles/bundle*.js,*/kalite/*/js/i18n/*.js" ;\ @@ -123,6 +123,9 @@ dist: clean docs assets make clean-pyc python setup.py sdist --formats=$(format) python setup.py bdist_wheel + pip install -t kalite/packages/dist -r "requirements.txt" + rm -rf kalite/packages/dist/*.dist-info # pip installs from PyPI will complain if we have more than one dist-info directory. + rm -r kalite/packages/dist/man kolibri/dist/bin || true # remove the two folders introduced by pip 10 python setup.py sdist --formats=$(format) --static python setup.py bdist_wheel --static --no-clean ls -l dist diff --git a/docs/index.rst b/docs/index.rst index 08e162534a..8a41fde59f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -46,6 +46,15 @@ Khan Academy content (videos and exercises) from a local server, with points and progress-tracking, without needing internet connectivity. +Translations +------------ + +KA Lite is released in many languages, each with its own subset of Khan Academy contents translated: Polish, Georgian, German, Spanish, Xhosa, Indonesian, Burmese, Swahili, Tamil, Hindi, Portuguese (Brazilian and Portuguese), Bulgarian, Danish, Arabic, Bengali, Kannada, and French. See :ref:`adding_languages`. + +Documentation: This documentation is also available +`in French (PDF) `__. + + Get involved! ------------- diff --git a/docs/installguide/advanced.rst b/docs/installguide/advanced.rst index 76b5d28478..f808f2fd11 100644 --- a/docs/installguide/advanced.rst +++ b/docs/installguide/advanced.rst @@ -62,7 +62,7 @@ and if you are connected to the internet, this will also give you automatic upda To add the PPA as a repository on an apt-based system, you need to ensure that a few libraries are present, and then add our repository and the public key that packages are signed with:: - sudo apt-get install software-properties-common python-software-properties + sudo apt-get install dirmngr sudo su -c 'echo "deb http://ppa.launchpad.net/learningequality/ka-lite/ubuntu xenial main" > /etc/apt/sources.list.d/ka-lite.list' sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 74F88ADB3194DD81 sudo apt-get update diff --git a/docs/installguide/release_notes.rst b/docs/installguide/release_notes.rst index 9c33f65ff0..387e86ed38 100644 --- a/docs/installguide/release_notes.rst +++ b/docs/installguide/release_notes.rst @@ -9,6 +9,16 @@ to read the release notes. ``0.15.x`` to ``0.17.x`` is not guaranteed to work. +0.17.5 +------ + +Bug fixes +^^^^^^^^^ + +* Slow download using ``retrievecontentpack`` on Raspberry Pi :url-issue:`5575` +* Customized content packs may fail depending on the Zip program used :url-issue:`5476` + + 0.17.4 ------ @@ -16,6 +26,7 @@ Added ^^^^^ * Progress displayed while downloading content packs :url-issue:`5356` +* Customizable welcome message setting ``KALITE_WELCOME_MESSAGE`` displayed to admin users on first login :url-issue:`5522` Bug fixes ^^^^^^^^^ @@ -27,10 +38,6 @@ Bug fixes * Also delete content database when deleting a content pack :url-issue:`5545` * Simplified login is now working when there are 1,000 or more users registered in a facility. :url-issue:`5523` -New Features -^^^^^^^^^^^^ - -* Customizable welcome message setting ``KALITE_WELCOME_MESSAGE`` displayed to admin users on first login :url-issue:`5522` Developers ^^^^^^^^^^ @@ -41,6 +48,13 @@ Developers Known issues ^^^^^^^^^^^^ +* The English **Math** topic (top-level) contains a number of misplaced + subtopics in the 0.17 series. The problem has propagated from upstream data. + The misplacement clutters/obstructs the navigation for students, and it has + been a priority to fix this in 0.18, released shortly after 0.17.4. + But it's recommended to upgrade to 0.18 and download new video contents + following from the new content pack (~450 videos). See below for tips on how + to install a content pack from a different KA Lite release. :url-issue:`5561` * It isn't possible to cancel video downloads if a video is downloading while the connection is switched off. * **Chrome 55-56** has issues scrolling the menus on touch devices. Upgrading to Chrome 57 fixes this. :url-issue:`5407` @@ -51,6 +65,17 @@ Known issues * A limited number of exercises with radio buttons have problems displaying :url-issue:`5172` +.. tip:: + It is possible to install a 0.16 content pack in a 0.17 release as a + work-around. Download a specific content pack and then install it from + command-line. Example:: + + # Download content pack to "en_0.16.zip" + $ wget -r -O en_0.16.zip http://pantry.learningequality.org/downloads/ka-lite/0.16/content/contentpacks/en.zip + # Install "en_0.16.zip" + $ kalite manage retrievecontentpack local en en_0.16.zip + + 0.17.3 ------ diff --git a/docs/usermanual/userman_admin.rst b/docs/usermanual/userman_admin.rst index 10e08303e7..ee2f5b18f6 100644 --- a/docs/usermanual/userman_admin.rst +++ b/docs/usermanual/userman_admin.rst @@ -495,6 +495,7 @@ After you copied in the new video files or changed the ``CONTENT_ROOT`` path, yo 4. Once the scan is completed, video content will be ready for Learners to watch! +.. _adding_languages: Adding Languages ---------------- @@ -675,6 +676,8 @@ _________________ institutions where permissions should be reserved for admins. * ``USER_LOG_MAX_RECORDS_PER_USER = (default = 0 [disabled], -1=unlimited logs)`` In order to keep local data in the ``UserLog`` model, detailing usage, you can choose the number of ``UserLog`` objects that you wish to retain. These objects are not sync'ed. +* ``SIMPLIFIED_LOGIN = (default = False)``. + Switches off passwords in the Learner's login modal. Online synchronization diff --git a/kalite/distributed/management/commands/retrievecontentpack.py b/kalite/distributed/management/commands/retrievecontentpack.py index f936b75e37..6a2ef99653 100644 --- a/kalite/distributed/management/commands/retrievecontentpack.py +++ b/kalite/distributed/management/commands/retrievecontentpack.py @@ -23,6 +23,7 @@ from peewee import SqliteDatabase from kalite.topic_tools.content_models import Item, AssessmentItem from requests.exceptions import ConnectionError, HTTPError +from fle_utils.django_utils.command import LocaleAwareCommand logger = logging.getLogger(__name__) @@ -41,7 +42,19 @@ class Command(UpdatesStaticCommand): """ - option_list = UpdatesStaticCommand.option_list + ( + + option_list = LocaleAwareCommand.option_list + ( + make_option( + "", "--background", + action="store_true", + dest="background", + default=False, + help=( + "Run the command with job scheduler and database-backed " + "progress. Used when you retrieve a content pack before " + "initializing KA Lite's db." + ) + ), make_option( "", "--template", action="store_true", @@ -147,6 +160,9 @@ def download(self, *args, **options): lang = args[1] + # This rather hacky callback will add progress to the database-backed + # progress monitoriing, which is terrible for performance on for + # instance Raspberry Pi devices. def download_callback(fraction): percent = int(fraction * 100) self.update_stage( @@ -223,7 +239,10 @@ def extract_subtitles(zf, lang): ensure_dir(SUBTITLE_DEST_DIR) - subtitles = (s for s in zf.namelist() if SUBTITLE_ZIP_DIR in s) + def is_subtitle_file(s): + return SUBTITLE_ZIP_DIR in s and len(s) > len(SUBTITLE_ZIP_DIR) + + subtitles = (s for s in zf.namelist() if is_subtitle_file(s)) for subtitle in subtitles: # files inside zipfiles may come with leading directories in their diff --git a/kalite/testing/testrunner.py b/kalite/testing/testrunner.py index 700aef9afe..e44e73c7d8 100644 --- a/kalite/testing/testrunner.py +++ b/kalite/testing/testrunner.py @@ -179,7 +179,7 @@ def build_suite(self, test_labels, extra_tests, **kwargs): browser.quit() if not database_exists() or os.path.getsize(database_path()) < 1024 * 1024: - call_command("retrievecontentpack", "empty", "en", force=True, foreground=True) + call_command("retrievecontentpack", "empty", "en", force=True) logging.info("Successfully setup content database") else: logging.info("Content database already exists") diff --git a/kalite/updates/api_views.py b/kalite/updates/api_views.py index 52f39e4780..47d8291574 100644 --- a/kalite/updates/api_views.py +++ b/kalite/updates/api_views.py @@ -205,7 +205,7 @@ def start_languagepack_download(request): data = json.loads(request.raw_post_data) # Django has some weird post processing into request.POST, so use .body lang_code = lcode_to_ietf(data['lang']) - call_command_async('retrievecontentpack', 'download', lang_code) + call_command_async('retrievecontentpack', 'download', lang_code, background=True) return JsonResponseMessageSuccess(_("Successfully started language pack download for %(lang_name)s.") % {"lang_name": get_language_name(lang_code)}) diff --git a/kalite/updates/management/utils.py b/kalite/updates/management/utils.py index dc82d0bc3c..37553d37c1 100644 --- a/kalite/updates/management/utils.py +++ b/kalite/updates/management/utils.py @@ -29,7 +29,11 @@ class UpdatesCommand(LocaleAwareCommand): """ Abstract class for sharing code across Dynamic and Static versions """ + option_list = LocaleAwareCommand.option_list + ( + # WARNING: Because of issues having foreground as a default option, we + # are overwriting this with a --background option in + # retrievecontentpack make_option( "", "--foreground", action="store_true", @@ -39,7 +43,8 @@ class UpdatesCommand(LocaleAwareCommand): "Run the command without job scheduler and database-backed " "progress. Used when you retrieve a content pack before " "initializing KA Lite's db." - )), + ) + ), ) def __init__(self, process_name=None, *args, **kwargs): @@ -53,9 +58,10 @@ def setup(self, options): # Should be set by command's handle() method since there's no clear # call structure defined. Respected by the methods that update stuff # in the database. - self.foreground = options["foreground"] + self.foreground = options.get("foreground", False) + self.background = options.get("background", True) - if not self.foreground: + if not self.foreground and self.background: self.progress_log = UpdateProgressLog.get_active_log(process_name=self.process_name) else: self.progress_log = None diff --git a/kalite/version.py b/kalite/version.py index 8717af891d..1d478b956d 100644 --- a/kalite/version.py +++ b/kalite/version.py @@ -3,7 +3,7 @@ # Must also be of the form N.N.N for internal use, where N is a non-negative integer MAJOR_VERSION = "0" MINOR_VERSION = "17" -PATCH_VERSION = "4" +PATCH_VERSION = "5" VERSION = "%s.%s.%s" % (MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION) SHORTVERSION = "%s.%s" % (MAJOR_VERSION, MINOR_VERSION) diff --git a/setup.py b/setup.py index 5464f9d2ed..4982a1eef2 100644 --- a/setup.py +++ b/setup.py @@ -207,59 +207,10 @@ def run(self): if STATIC_BUILD: sys.stderr.write( - "This is a static build... invoking pip to put static dependencies in " - "kalite/packages/dist/\n" + "This is a static build... we hope you invoked this build through the " + "'make dist' target, because this populates kalite/packages/dist" ) - STATIC_DIST_PACKAGES_DOWNLOAD_CACHE = os.path.join(where_am_i, '.pip-downloads') - STATIC_DIST_PACKAGES_TEMP = os.path.join(where_am_i, '.pip-temp') - - # Create directory where dynamically created dependencies are put - if not os.path.exists(STATIC_DIST_PACKAGES_DOWNLOAD_CACHE): - os.mkdir(STATIC_DIST_PACKAGES_DOWNLOAD_CACHE) - - # Should remove the temporary directory always - if not NO_CLEAN and os.path.exists(STATIC_DIST_PACKAGES_TEMP): - print("Removing previous temporary sources for pip {}".format(STATIC_DIST_PACKAGES_TEMP)) - shutil.rmtree(STATIC_DIST_PACKAGES_TEMP) - - # Install from pip - - # Code modified from this example: - # http://threebean.org/blog/2011/06/06/installing-from-pip-inside-python-or-a-simple-pip-api/ - import pip.commands.install - - # Ensure we get output from pip - enable_log_to_stdout('pip.commands.install') - - def install_distributions(distributions): - command = pip.commands.install.InstallCommand() - opts, ___ = command.parser.parse_args([]) - opts.target_dir = STATIC_DIST_PACKAGES - opts.build_dir = STATIC_DIST_PACKAGES_TEMP - opts.download_cache = STATIC_DIST_PACKAGES_DOWNLOAD_CACHE - opts.isolated = True - opts.ignore_installed = True - opts.compile = False - opts.ignore_dependencies = False - # This is deprecated and will disappear in Pip 10 - opts.use_wheel = False - # The below is not an option, then we skip mimeparse - # opts.no_binary = ':all:' # Do not use any binary files (whl) - opts.no_clean = NO_CLEAN - command.run(opts, distributions) - - # Install requirements into kalite/packages/dist - if DIST_BUILDING_COMMAND: - install_distributions(RAW_REQUIREMENTS) - - # Now remove Django because it's bundled. It gets installed as an egg - # so unfortunately, the path is a bit dependent on the specific - # version installed (we pinned it for reliability in requirements.txt) - shutil.rmtree( - os.path.join(STATIC_DIST_PACKAGES, "Django-1.5.12-py2.7.egg-info"), - ) - # It's not a build command with --static or it's not a build command at all else: