diff --git a/kalite/distributed/management/commands/retrievecontentpack.py b/kalite/distributed/management/commands/retrievecontentpack.py index ebd6029d5a..817e93400d 100644 --- a/kalite/distributed/management/commands/retrievecontentpack.py +++ b/kalite/distributed/management/commands/retrievecontentpack.py @@ -13,8 +13,9 @@ from fle_utils.general import ensure_dir +from kalite.contentload import settings as content_settings from kalite.i18n.base import lcode_to_django_lang, get_po_filepath, get_locale_path, \ - update_jsi18n_file + update_jsi18n_file, get_srt_path as get_subtitle_path from kalite.topic_tools import settings from kalite.updates.management.commands.classes import UpdatesStaticCommand @@ -82,7 +83,9 @@ def process_content_pack(self, zf, lang): extract_catalog_files(zf, lang) update_jsi18n_file(lang) extract_content_db(zf, lang) - extract_content_pack_metadata(zf, lang) + extract_subtitles(zf, lang) + extract_content_pack_metadata(zf, lang) # always extract to the en lang + extract_assessment_items(zf, "en") self.next_stage(_("Looking for available content items.")) call_command("annotate_content_items", language=lang) @@ -140,3 +143,54 @@ def extract_content_db(zf, lang): with open(content_db_path, "wb") as f: dbfobj = zf.open("content.db") shutil.copyfileobj(dbfobj, f) + + +def extract_subtitles(zf, lang): + SUBTITLE_DEST_DIR = get_subtitle_path(lang_code=lang) + SUBTITLE_ZIP_DIR = "subtitles/" + + ensure_dir(SUBTITLE_DEST_DIR) + + subtitles = (s for s in zf.namelist() if SUBTITLE_ZIP_DIR in s) + + for subtitle in subtitles: + # files inside zipfiles may come with leading directories in their + # names, like subtitles/hotdog.vtt. We'll only want the actual filename + # (hotdog.vtt) when extracting as that's what KA Lite expects. + + subtitle_filename = os.path.basename(subtitle) + subtitle_dest_path = os.path.join(SUBTITLE_DEST_DIR, subtitle_filename) + + subtitle_fileobj = zf.open(subtitle) + + with open(subtitle_dest_path, "w") as dest_fileobj: + shutil.copyfileobj(subtitle_fileobj, dest_fileobj) + + +def extract_assessment_items(zf, lang): + assessment_zip_dir = "assessment_resources/" + + channel = "khan" + + assessment_dest_dir = os.path.join(content_settings.ASSESSMENT_ITEM_ROOT, channel) + + ensure_dir(assessment_dest_dir) + + items = (s for s in zf.namelist() if assessment_zip_dir in s) + + for item in items: + # files inside zipfiles may come with leading directories in their + # names, like subtitles/hotdog.vtt. We'll only want the actual filename + # (hotdog.vtt) when extracting as that's what KA Lite expects. + + filename = os.path.basename(item) + subfolder = filename[:3] + assessment_subfolder = os.path.join(assessment_dest_dir, subfolder) + + ensure_dir(assessment_subfolder) + + zip_item_fileobj = zf.open(item) + + assessment_dest_path = os.path.join(assessment_subfolder, filename) + with open(assessment_dest_path, "w") as item_fileobj: + shutil.copyfileobj(zip_item_fileobj, item_fileobj) diff --git a/kalite/distributed/management/commands/setup.py b/kalite/distributed/management/commands/setup.py index f31ce501a7..f26ea03f29 100644 --- a/kalite/distributed/management/commands/setup.py +++ b/kalite/distributed/management/commands/setup.py @@ -5,7 +5,7 @@ - Run migrations - Find and relocate obsolete user and data files - if interactive: - - Download and unpack assessment items + - Download and unpack the english content pack, containing assessment items - Create super user account - Run 'kalite start' """ @@ -29,19 +29,17 @@ import kalite from kalite.contentload.settings import KHAN_ASSESSMENT_ITEM_ROOT, OLD_ASSESSMENT_ITEMS_LOCATION -from kalite.topic_tools.settings import CHANNEL from fle_utils.config.models import Settings from fle_utils.general import get_host_name from fle_utils.platforms import is_windows -from kalite.version import VERSION, SHORTVERSION +from kalite.distributed.management.commands.retrievecontentpack import CONTENT_PACK_URL_TEMPLATE from kalite.facility.models import Facility +from kalite.version import VERSION, SHORTVERSION from securesync.models import Device import warnings - -# for extracting assessment item resources -ASSESSMENT_ITEMS_ZIP_URL = "https://learningequality.org/downloads/ka-lite/{version}/content/{channel}_assessment.zip".format(version=SHORTVERSION, channel=CHANNEL) +CONTENTPACK_URL = CONTENT_PACK_URL_TEMPLATE.format(version=SHORTVERSION, code="en") def raw_input_yn(prompt): @@ -139,7 +137,7 @@ def validate_filename(filename): return False def find_recommended_file(): - filename_guess = "{channel}_assessment.zip".format(channel=CHANNEL) + filename_guess = "en.zip" curdir = os.path.abspath(os.curdir) pardir = os.path.abspath(os.path.join(curdir, os.pardir)) while curdir != pardir: @@ -152,8 +150,9 @@ def find_recommended_file(): return "" recommended_filename = find_recommended_file() - prompt = "Please enter the filename of the assessment items package you have downloaded (%s): " % recommended_filename + prompt = "Please enter the filename of the content pack you have downloaded (%s): " % recommended_filename filename = raw_input(prompt) + filename = os.path.expanduser(filename) if not filename: filename = recommended_filename while not validate_filename(filename): @@ -204,12 +203,12 @@ class Command(BaseCommand): action='store_true', dest='force-assessment-item-dl', default=False, - help='Downloads assessment items from the url specified by ASSESSMENT_ITEMS_ZIP_URL, without interaction'), + help='Downloads content pack from the url specified by CONTENTPACK_URL, without interaction'), make_option('-i', '--no-assessment-items', action='store_true', dest='no-assessment-items', default=False, - help='Skip all steps associating with assessment item downloading or the assessment item database'), + help='Skip all steps associating with content pack downloading or the content database'), make_option('-g', '--git-migrate', action='store', dest='git_migrate_path', @@ -387,12 +386,12 @@ def handle(self, *args, **options): call_command("migrate", merge=True, verbosity=options.get("verbosity")) Settings.set("database_version", VERSION) - # download assessment items + # download the english content pack # This can take a long time and lead to Travis stalling. None of this # is required for tests, and does not apply to the central server. if options.get("no-assessment-items", False): - logging.warning("Skipping assessment item downloading and configuration.") + logging.warning("Skipping content pack downloading and configuration.") else: @@ -411,7 +410,7 @@ def handle(self, *args, **options): if writable_assessment_items and options['force-assessment-item-dl']: call_command( - "unpack_assessment_zip", ASSESSMENT_ITEMS_ZIP_URL) + "retrievecontentpack", "download", "en") elif options['force-assessment-item-dl']: raise RuntimeError( "Got force-assessment-item-dl but directory not writable") @@ -423,20 +422,23 @@ def handle(self, *args, **options): print( "If you have already downloaded the assessment items package, you can specify the file in the next step.") print("Otherwise, we will download it from {url}.".format( - url=ASSESSMENT_ITEMS_ZIP_URL)) + url=CONTENTPACK_URL)) - if raw_input_yn("Do you wish to download the assessment items package now?"): - ass_item_filename = ASSESSMENT_ITEMS_ZIP_URL - elif raw_input_yn("Have you already downloaded the assessment items package?"): + if raw_input_yn("Do you wish to download the content pack now?"): + ass_item_filename = CONTENTPACK_URL + retrieval_method = "download" + elif raw_input_yn("Have you already downloaded the content pack?"): ass_item_filename = get_assessment_items_filename() + retrieval_method = "local" else: ass_item_filename = None + retrieval_method = "local" if not ass_item_filename: logging.warning( - "No assessment items package file given. You will need to download and unpack it later.") + "No content pack given. You will need to download and unpack it later.") else: - call_command("unpack_assessment_zip", ass_item_filename) + call_command("retrievecontentpack", retrieval_method, "en", ass_item_filename) elif options['interactive']: logging.warning( diff --git a/kalite/distributed/templates/distributed/base.html b/kalite/distributed/templates/distributed/base.html index 38543285a1..dfcc053b72 100644 --- a/kalite/distributed/templates/distributed/base.html +++ b/kalite/distributed/templates/distributed/base.html @@ -60,7 +60,7 @@ STATIC_URL : "{% static "" %}", - ALL_ASSESSMENT_ITEMS_URL : "{% url 'api_dispatch_list' resource_name='assessment_item' %}", + ALL_ASSESSMENT_ITEMS_URL : "{% url 'assessment_item' assessment_item='' %}", GET_VIDEO_LOGS_URL : "{% url 'api_dispatch_list' resource_name='videolog' %}", GET_EXERCISE_LOGS_URL : "{% url 'api_dispatch_list' resource_name='exerciselog' %}", GET_CONTENT_LOGS_URL : "{% url 'api_dispatch_list' resource_name='contentlog' %}", diff --git a/kalite/distributed/tests/browser_tests/__init__.py b/kalite/distributed/tests/browser_tests/__init__.py index e6f146252b..ad209d3b99 100644 --- a/kalite/distributed/tests/browser_tests/__init__.py +++ b/kalite/distributed/tests/browser_tests/__init__.py @@ -1,6 +1,4 @@ from .base import * -# from .coachreports import * from .control_panel import * from .distributed import * from .knowledge_map import * -from .language_packs import * diff --git a/kalite/distributed/tests/browser_tests/control_panel.py b/kalite/distributed/tests/browser_tests/control_panel.py index 093a8ee99f..a660e763fd 100644 --- a/kalite/distributed/tests/browser_tests/control_panel.py +++ b/kalite/distributed/tests/browser_tests/control_panel.py @@ -84,7 +84,7 @@ def test_ungrouped_number_displays_correctly(self): """ Ungrouped # of students wasn't displaying correctly, see: https://github.com/learningequality/ka-lite/pull/2230 In particular it seems to have only occurred when a non-english language was set, so this test tried to - mock a language pack download -- but that makes the test dependent on languagepackdownload.py details :( + mock a language pack download """ facility = self.facility params = { diff --git a/kalite/distributed/tests/browser_tests/language_packs.py b/kalite/distributed/tests/browser_tests/language_packs.py deleted file mode 100644 index 4b179001ce..0000000000 --- a/kalite/distributed/tests/browser_tests/language_packs.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -These use a web-browser, along selenium, to simulate user actions. -""" -import os -import time -import urllib - -import mock -from django.conf import settings -from django.core.management import call_command -from django.test.utils import override_settings -from django.utils import unittest -from kalite.i18n.base import get_installed_language_packs -from kalite.testing.base import KALiteBrowserTestCase -from kalite.testing.mixins.browser_mixins import BrowserActionMixins -from kalite.testing.mixins.django_mixins import CreateAdminMixin -from selenium.webdriver.common.keys import Keys - - -logging = settings.LOG - - -@unittest.skipIf(getattr(settings, 'HEADLESS', None), "Doesn't work on HEADLESS.") -class LanguagePackTest(CreateAdminMixin, BrowserActionMixins, KALiteBrowserTestCase): - - def setUp(self): - self.admin_data = {"username": "admin", "password": "admin"} - self.admin = self.create_admin(**self.admin_data) - - super(LanguagePackTest, self).setUp() - - def is_language_installed(self, lang_code, force_reload=True): - return lang_code in get_installed_language_packs(force=force_reload) - - # @unittest.skipIf(settings.RUNNING_IN_TRAVIS, "Skip tests that fail when run on Travis, but succeed locally.") - # @mock.patch.object(urllib, 'urlretrieve') - # @override_settings(DEBUG=False) - # def test_delete_language_pack(self, urlretrieve_method): - # ''' Test to check whether a language pack is deleted successfully or not ''' - # test_zip_filepath = os.path.join(os.path.dirname(__file__), 'de.zip') - # urlretrieve_method.return_value = [test_zip_filepath, open(test_zip_filepath)] - # # Login as admin - # self.browser_login_admin(**self.admin_data) - - # # Delete the language pack - # if not self.is_language_installed("de"): - # call_command("languagepackdownload", lang_code="de") - - # self.register_device() - # language_url = self.reverse("update_languages") - # self.browse_to(language_url) - # time.sleep(1) - # self.browser.find_element_by_css_selector(".delete-language-button > button[value='de']").click() - # time.sleep(0.5) - # self.browser_send_keys(Keys.RETURN) - # time.sleep(1) - # self.assertFalse(self.is_language_installed("de")) diff --git a/kalite/i18n/tests/__init__.py b/kalite/i18n/tests/__init__.py index 82ff7a6742..e69de29bb2 100644 --- a/kalite/i18n/tests/__init__.py +++ b/kalite/i18n/tests/__init__.py @@ -1,2 +0,0 @@ -from base import * -from browser_tests import * diff --git a/kalite/i18n/tests/base.py b/kalite/i18n/tests/base.py deleted file mode 100644 index f9bacb61a0..0000000000 --- a/kalite/i18n/tests/base.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -Base classes to help test i18n functions -""" -import logging -import os -import urllib -from mock import patch - -from django.core.management import call_command -from django.test import TestCase - -from ..base import get_installed_language_packs, delete_language - - -class I18nTestCase(TestCase): - - def is_language_installed(self, lang_code, force_reload=True): - return lang_code in get_installed_language_packs(force=force_reload) - - @patch.object(urllib, 'urlretrieve') - def install_language(self, lang_code, urlretrieve_method): - test_zip_filepath = os.path.join(os.path.dirname(__file__), 'testzips', '%s.zip' % lang_code) - urlretrieve_method.return_value = [test_zip_filepath, open(test_zip_filepath)] - - if not self.is_language_installed(lang_code): - call_command('languagepackdownload', lang_code=lang_code) - - def install_languages(self): - # install TEST_LANGUAGES, if defined - if not self.TEST_LANGUAGES: - logging.debug("self.TEST_LANGUAGES not defined. Not installing any language.") - else: - logging.disable(logging.ERROR) # silence langpack installation logs - - for lang in self.TEST_LANGUAGES: - self.install_language(lang) - - logging.disable(logging.NOTSET) # reactivate logs again - - - def uninstall_languages(self): - logging.disable(logging.ERROR) - - for lang in self.TEST_LANGUAGES: - delete_language(lang) - - logging.disable(logging.NOTSET) diff --git a/kalite/i18n/tests/browser_tests.py b/kalite/i18n/tests/browser_tests.py deleted file mode 100644 index af3a080445..0000000000 --- a/kalite/i18n/tests/browser_tests.py +++ /dev/null @@ -1,30 +0,0 @@ -# encoding: UTF-8 -from django.conf import settings -from django.utils import unittest - -from kalite.testing.base import KALiteBrowserTestCase -from kalite.testing.mixins.browser_mixins import BrowserActionMixins -from kalite.testing.mixins.django_mixins import CreateAdminMixin -from kalite.i18n.tests.base import I18nTestCase - - -class BrowserLanguageSwitchingTests(BrowserActionMixins, CreateAdminMixin, I18nTestCase, KALiteBrowserTestCase): - - TEST_LANGUAGES = ['it', 'pt-BR'] - - def setUp(self): - self.foreign_language_homepage_text_mappings = { - 'it': "gratuita per chiunque ovunque", - 'pt-BR': "Uma educação sem salas de aula para qualquer um em qualquer lugar", - } - - self.admin_data = {"username": "admin", "password": "admin"} - self.admin = self.create_admin(**self.admin_data) - assert sorted(self.TEST_LANGUAGES) == sorted(self.foreign_language_homepage_text_mappings.keys()) - - super(BrowserLanguageSwitchingTests, self).setUp() - self.install_languages() - - def tearDown(self): - self.uninstall_languages() - super(BrowserLanguageSwitchingTests, self).tearDown() diff --git a/kalite/i18n/tests/create_dummy_language_pack_tests.py b/kalite/i18n/tests/create_dummy_language_pack_tests.py index 760b78d1c6..2b72ec17f5 100644 --- a/kalite/i18n/tests/create_dummy_language_pack_tests.py +++ b/kalite/i18n/tests/create_dummy_language_pack_tests.py @@ -1,4 +1,3 @@ -import accenting import requests import zipfile from cStringIO import StringIO diff --git a/kalite/i18n/tests/testzips/de.zip b/kalite/i18n/tests/testzips/de.zip deleted file mode 100644 index 1ce9457136..0000000000 Binary files a/kalite/i18n/tests/testzips/de.zip and /dev/null differ diff --git a/kalite/i18n/tests/testzips/it.zip b/kalite/i18n/tests/testzips/it.zip deleted file mode 100644 index 345da4a26c..0000000000 Binary files a/kalite/i18n/tests/testzips/it.zip and /dev/null differ diff --git a/kalite/i18n/tests/testzips/pt-BR.zip b/kalite/i18n/tests/testzips/pt-BR.zip deleted file mode 100644 index 2a71979654..0000000000 Binary files a/kalite/i18n/tests/testzips/pt-BR.zip and /dev/null differ diff --git a/kalite/main/api_resources.py b/kalite/main/api_resources.py index c5ff97f19a..f1c8d2d3a9 100644 --- a/kalite/main/api_resources.py +++ b/kalite/main/api_resources.py @@ -1,18 +1,12 @@ -import json from tastypie import fields -from tastypie.resources import ModelResource, Resource -from tastypie.exceptions import NotFound -from django.conf.urls import url -from django.conf import settings +from tastypie.resources import ModelResource from .models import VideoLog, ExerciseLog, AttemptLog, ContentLog, ContentRating -from kalite.topic_tools.content_models import get_assessment_item_data, AssessmentItem from kalite.shared.api_auth.auth import UserObjectsOnlyAuthorization from kalite.facility.api_resources import FacilityUserResource - class ExerciseLogResource(ModelResource): user = fields.ForeignKey(FacilityUserResource, 'user') @@ -89,43 +83,6 @@ class Meta: authorization = UserObjectsOnlyAuthorization() -class AssessmentItemResource(ModelResource): - class Meta: - resource_name = 'assessment_item' - - def prepend_urls(self): - return [ - url(r"^(?P%s)/(?P[\w\d_.-]+)/$" % self._meta.resource_name, - self.wrap_view('dispatch_detail'), - name="api_dispatch_detail"), - ] - - - def obj_get(self, bundle, **kwargs): - id = kwargs.get("id") - assessment_item = get_assessment_item_data(channel=getattr(bundle.request, "channel", "khan"), language=getattr(bundle.request, "language", "en"), assessment_item_id=id) - if assessment_item: - return AssessmentItem(**assessment_item) - else: - raise NotFound('AssessmentItem with id %s not found' % id) - - - def obj_create(self, bundle, **kwargs): - raise NotImplemented("Operation not implemented yet for assessment_items.") - - def obj_update(self, bundle, **kwargs): - raise NotImplemented("Operation not implemented yet for assessment_items.") - - def obj_delete_list(self, bundle, **kwargs): - raise NotImplemented("Operation not implemented yet for assessment_items.") - - def obj_delete(self, bundle, **kwargs): - raise NotImplemented("Operation not implemented yet for assessment_items.") - - def rollback(self, bundles): - raise NotImplemented("Operation not implemented yet for assessment_items.") - - class ContentRatingResource(ModelResource): user = fields.ForeignKey(FacilityUserResource, 'user') diff --git a/kalite/main/api_urls.py b/kalite/main/api_urls.py index 7a01944b06..5635ae6f4e 100755 --- a/kalite/main/api_urls.py +++ b/kalite/main/api_urls.py @@ -7,7 +7,7 @@ from django.conf.urls import include, patterns, url from .api_resources import VideoLogResource, ExerciseLogResource, AttemptLogResource, ContentLogResource,\ - AssessmentItemResource, ContentRatingResource + ContentRatingResource urlpatterns = patterns(__package__ + '.api_views', @@ -16,10 +16,9 @@ url(r'^', include(ExerciseLogResource().urls)), url(r'^', include(AttemptLogResource().urls)), url(r'^', include(ContentLogResource().urls)), - # Retrieve assessment item data to render front-end Perseus Exercises - url(r'^', include(AssessmentItemResource().urls)), url(r'^', include(ContentRatingResource().urls)), - + + url(r'^assessment_item/(?P.*)/$', 'assessment_item', {}, 'assessment_item'), url(r'^content_recommender/?$', 'content_recommender', {}, 'content_recommender'), # A flat data structure for building a graphical knowledge map diff --git a/kalite/main/api_views.py b/kalite/main/api_views.py index 9d3da8981a..1d1aae30b1 100755 --- a/kalite/main/api_views.py +++ b/kalite/main/api_views.py @@ -12,7 +12,7 @@ from fle_utils.internet.decorators import api_handle_error_with_json from fle_utils.internet.classes import JsonResponse, JsonResponseMessageError -from kalite.topic_tools.content_models import get_topic_nodes, get_content_item, search_topic_nodes +from kalite.topic_tools.content_models import get_topic_nodes, get_content_item, search_topic_nodes, get_assessment_item_data from kalite.topic_tools.content_recommendation import get_resume_recommendations, get_next_recommendations, get_explore_recommendations from kalite.facility.models import FacilityUser from kalite.distributed.api_views import get_messages_for_api_calls @@ -104,3 +104,11 @@ def set_bool_flag(flag_name, rec_dict): explore_recommendations = [set_bool_flag("explore", rec) for rec in get_explore_recommendations(user, request)] if explore else [] return JsonResponse(resume_recommendations + next_recommendations + explore_recommendations) + + +@api_handle_error_with_json +def assessment_item(request, assessment_item): + assessment_item = get_assessment_item_data(channel=getattr(request, "channel", "khan"), + language=getattr(request, "language", "en"), + assessment_item_id=assessment_item) + return JsonResponse(assessment_item) diff --git a/kalite/topic_tools/annotate.py b/kalite/topic_tools/annotate.py index d30302d448..0fd65ff3d7 100644 --- a/kalite/topic_tools/annotate.py +++ b/kalite/topic_tools/annotate.py @@ -59,10 +59,10 @@ def update_content_availability(content_list, language="en", channel="khan"): if content.get("kind") == "Exercise": - # Databases have been pre-filtered to only contain existing exercises + # Databases have been pre-filtered to only contain existing exercises. + # Exercises have been pre-marked as available as well. # Assume if the assessment items have been downloaded, then everything is hunky dory. - if os.path.exists(contentload_settings.KHAN_ASSESSMENT_ITEM_VERSION_PATH): - update["available"] = True + continue elif content.get("kind") == "Topic": # Ignore topics, as we only want to update their availability after we have updated the rest. diff --git a/kalite/topic_tools/tests/__init__.py b/kalite/topic_tools/tests/__init__.py index 76a66b3976..e16a370538 100644 --- a/kalite/topic_tools/tests/__init__.py +++ b/kalite/topic_tools/tests/__init__.py @@ -2,5 +2,4 @@ from helper_tests import * from next_tests import * from resume_tests import * -from annotate_tests import * from content_models_tests import UpdateItemTestCase, ContentModelsTestCase, ContentModelRegressionTestCase diff --git a/kalite/topic_tools/tests/annotate_tests.py b/kalite/topic_tools/tests/annotate_tests.py index 36696c1787..ef776ddb1a 100644 --- a/kalite/topic_tools/tests/annotate_tests.py +++ b/kalite/topic_tools/tests/annotate_tests.py @@ -69,7 +69,7 @@ def test_update_content_availability_false(self): with Using(self.db, [Item]): actual = dict(update_content_availability([unparse_model_data(model_to_dict(self.item))])).get("thepath") # Update is only generated if changed from False to True, not from False to False, so should return None. - assert not actual.get("available") + assert not actual try: os.rename(self.version_path + ".bak", self.version_path) diff --git a/kalite/updates/api_views.py b/kalite/updates/api_views.py index cc222d0043..0fb0a0d1ff 100644 --- a/kalite/updates/api_views.py +++ b/kalite/updates/api_views.py @@ -196,7 +196,7 @@ def start_languagepack_download(request): data = json.loads(request.raw_post_data) # Django has some weird post processing into request.POST, so use .body lang_code = lcode_to_ietf(data['lang']) - force_job('languagepackdownload', _("Language pack download"), lang_code=lang_code, locale=request.language) + call_command_async('retrievecontentpack', 'download', lang_code) return JsonResponseMessageSuccess(_("Successfully started language pack download for %(lang_name)s.") % {"lang_name": get_language_name(lang_code)}) diff --git a/kalite/updates/management/commands/languagepackdownload.py b/kalite/updates/management/commands/languagepackdownload.py deleted file mode 100644 index 335a47b088..0000000000 --- a/kalite/updates/management/commands/languagepackdownload.py +++ /dev/null @@ -1,214 +0,0 @@ -""" -Downloads a language pack, unzips the contents, then moves files accordingly -""" -import glob -import os -import shutil -import requests -import zipfile -from optparse import make_option -from StringIO import StringIO - -from django.conf import settings; logging = settings.LOG -import httplib - -from ... import REMOTE_VIDEO_SIZE_FILEPATH -from .classes import UpdatesStaticCommand -from fle_utils.chronograph.management.croncommand import CronCommand -from fle_utils.general import ensure_dir -from fle_utils.internet.download import callback_percent_proxy, download_file -from django.core.management import call_command -from django.core.management.base import CommandError -from django.utils.translation import ugettext as _ -from kalite.i18n.base import get_language_name, get_language_pack_url, \ - get_localized_exercise_dirpath, get_po_filepath, get_srt_path, \ - lcode_to_django_dir, lcode_to_ietf, update_jsi18n_file -from kalite.version import SHORTVERSION - - -class Command(UpdatesStaticCommand, CronCommand): - help = "Download language pack requested from central server" - - unique_option_list = ( - make_option('-l', '--language', - action='store', - dest='lang_code', - default=settings.LANGUAGE_CODE, - metavar="LANG_CODE", - help="Specify a particular language code to download language pack for."), - make_option('-s', '--software_version', - action='store', - dest='software_version', - default=SHORTVERSION, - metavar="SOFT_VERS", - help="Specify the software version to download a language pack for."), - make_option('-f', '--from-file', - action='store', - dest='file', - default=None, - help='Use the given zip file instead of fetching from the central server.'), - ) - - option_list = UpdatesStaticCommand.option_list + CronCommand.unique_option_list + unique_option_list - - stages = ( - "download_language_pack", - "unpack_language_pack", - "add_js18n_file", - "move_files", - "collectstatic", - "invalidate_caches", - ) - - def handle(self, *args, **options): - if settings.CENTRAL_SERVER: - raise CommandError("This must only be run on distributed servers server.") - - lang_code = lcode_to_ietf(options["lang_code"]) - lang_name = get_language_name(lang_code) - software_version = options["software_version"] - logging.info("Downloading language pack for lang_name=%s, software_version=%s" % (lang_name, software_version)) - - # Download the language pack - try: - if options['file']: - self.start(_("Using local language pack '%(filepath)s'") % {"filepath": options['file']}) - zip_filepath = options['file'] - else: - self.start(_("Downloading language pack '%(lang_code)s'") % {"lang_code": lang_code}) - zip_filepath = get_language_pack(lang_code, software_version, callback=self.cb) - - # Unpack into locale directory - self.next_stage(_("Unpacking language pack '%(lang_code)s'") % {"lang_code": lang_code}) - unpack_language(lang_code, zip_filepath=zip_filepath) - - # - self.next_stage(_("Creating static files for language pack '%(lang_code)s'") % {"lang_code": lang_code}) - update_jsi18n_file(lang_code) - - - self.next_stage(_("Moving files to their appropriate local disk locations.")) - move_dubbed_video_map(lang_code) - move_exercises(lang_code) - move_srts(lang_code) - move_video_sizes_file(lang_code) - - self.next_stage() - call_command("collectstatic", interactive=False) - - self.complete(_("Finished processing language pack %(lang_name)s.") % {"lang_name": get_language_name(lang_code)}) - except Exception as e: - self.cancel(stage_status="error", notes=_("Error: %(error_msg)s") % {"error_msg": unicode(e)}) - raise - - def cb(self, percent): - self.update_stage(stage_percent=percent / 100.) - -def get_language_pack(lang_code, software_version, callback): - """Download language pack for specified language""" - - lang_code = lcode_to_ietf(lang_code) - logging.info("Retrieving language pack: %s" % lang_code) - request_url = get_language_pack_url(lang_code, software_version) - logging.debug("Downloading zip from %s" % request_url) - - # aron: hack, download_file uses urllib.urlretrieve, which doesn't - # return a status code. So before we make the full request, we - # check first if the said lang pack url exists. If not, error out. - if requests.head(request_url).status_code == 404: - raise requests.exceptions.HTTPError("Language pack %s not found. Please double check that it exists." % lang_code) - - path, response = download_file(request_url, callback=callback_percent_proxy(callback)) - return path - -def unpack_language(lang_code, zip_filepath=None, zip_fp=None, zip_data=None): - """Unpack zipped language pack into locale directory""" - lang_code = lcode_to_django_dir(lang_code) - - logging.info("Unpacking new translations") - ensure_dir(get_po_filepath(lang_code=lang_code)) - - # # Unpack into temp dir - try: - z = zipfile.ZipFile(zip_fp or (zip_data and StringIO(zip_data)) or open(zip_filepath, "rb")) - except zipfile.BadZipfile as e: - # Need to add more information on the errror message. - # See http://stackoverflow.com/questions/6062576/adding-information-to-a-python-exception - raise type(e), type(e)(e.message + _("Language pack corrupted. Please try downloading the language pack again in a few minutes.")) - z.extractall(os.path.join(settings.USER_WRITABLE_LOCALE_DIR, lang_code)) - -def move_dubbed_video_map(lang_code): - lang_pack_location = os.path.join(settings.USER_WRITABLE_LOCALE_DIR, lang_code) - dubbed_video_dir = os.path.join(lang_pack_location, "dubbed_videos") - dvm_filepath = os.path.join(dubbed_video_dir, os.path.basename(settings.DUBBED_VIDEOS_MAPPING_FILEPATH)) - if not os.path.exists(dvm_filepath): - logging.error("Could not find downloaded dubbed video filepath: %s" % dvm_filepath) - else: - logging.debug("Moving dubbed video map to %s" % settings.DUBBED_VIDEOS_MAPPING_FILEPATH) - ensure_dir(os.path.dirname(settings.DUBBED_VIDEOS_MAPPING_FILEPATH)) - shutil.move(dvm_filepath, settings.DUBBED_VIDEOS_MAPPING_FILEPATH) - - logging.debug("Removing empty directory") - try: - shutil.rmtree(dubbed_video_dir) - except Exception as e: - logging.error("Error removing dubbed video directory (%s): %s" % (dubbed_video_dir, e)) - -def move_video_sizes_file(lang_code): - """ - This is no longer needed. See: - https://github.com/learningequality/ka-lite/issues/4538#issuecomment-144560505 - """ - return - -def move_exercises(lang_code): - lang_pack_location = os.path.join(settings.USER_WRITABLE_LOCALE_DIR, lang_code) - src_exercise_dir = os.path.join(lang_pack_location, "exercises") - dest_exercise_dir = get_localized_exercise_dirpath(lang_code) - - if not os.path.exists(src_exercise_dir): - logging.warn("Could not find downloaded exercises; skipping: %s" % src_exercise_dir) - else: - # Move over one at a time, to combine with any other resources that were there before. - ensure_dir(dest_exercise_dir) - all_exercise_files = glob.glob(os.path.join(src_exercise_dir, "*.html")) - logging.info("Moving %d downloaded exercises to %s" % (len(all_exercise_files), dest_exercise_dir)) - - for exercise_file in all_exercise_files: - shutil.move(exercise_file, os.path.join(dest_exercise_dir, os.path.basename(exercise_file))) - - logging.debug("Removing emtpy directory") - try: - shutil.rmtree(src_exercise_dir) - except Exception as e: - logging.error("Error removing dubbed video directory (%s): %s" % (src_exercise_dir, e)) - -def move_srts(lang_code): - """ - Srts live in the locale directory, but that's not exposed at any URL. So instead, - we have to move the srts out to /static/subtitles/[lang_code]/ - """ - lang_code_ietf = lcode_to_ietf(lang_code) - lang_code_django = lcode_to_django_dir(lang_code) - - subtitles_static_dir = os.path.join(settings.USER_STATIC_FILES, "subtitles") - src_dir = os.path.join(settings.USER_WRITABLE_LOCALE_DIR, lang_code_django, "subtitles") - dest_dir = get_srt_path(lang_code_django) - ensure_dir(dest_dir) - - lang_subtitles = glob.glob(os.path.join(src_dir, "*.srt")) - logging.info("Moving %d subtitles from %s to %s" % (len(lang_subtitles), src_dir, dest_dir)) - - for fil in lang_subtitles: - srt_dest_path = os.path.join(dest_dir, os.path.basename(fil)) - if os.path.exists(srt_dest_path): - os.remove(srt_dest_path) # we're going to replace any srt with a newer version - shutil.move(fil, srt_dest_path) - - if not os.path.exists(src_dir): - logging.info("No subtitles for language pack %s" % lang_code) - elif os.listdir(src_dir): - logging.warn("%s is not empty; will not remove. Please check that all subtitles were moved." % src_dir) - else: - logging.info("Removing empty source directory (%s)." % src_dir) - shutil.rmtree(src_dir) diff --git a/kalite/updates/static/js/updates/bundle_modules/update_languages.js b/kalite/updates/static/js/updates/bundle_modules/update_languages.js index b3741aaa62..debd328c6e 100644 --- a/kalite/updates/static/js/updates/bundle_modules/update_languages.js +++ b/kalite/updates/static/js/updates/bundle_modules/update_languages.js @@ -32,8 +32,7 @@ function version_comparison(v1, v2) { function get_available_languages() { return api.doRequest(window.sessionModel.get("AVAILABLE_LANGUAGEPACK_URL"), null, { - cache: false, - dataType: "jsonp" + cache: false }).success(function(languages) { installable_languages = languages; display_languages(); diff --git a/kalite/updates/templates/updates/update_languages.html b/kalite/updates/templates/updates/update_languages.html index 4978f20d9a..c947050976 100644 --- a/kalite/updates/templates/updates/update_languages.html +++ b/kalite/updates/templates/updates/update_languages.html @@ -16,7 +16,7 @@