From e7ec79ec40a47482bd5a62718e372d469ecb9b19 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Mon, 20 Jul 2015 23:45:51 +0200 Subject: [PATCH 1/7] Have one single KEY_PREFIX --- docs/usermanual/userman_admin.rst | 7 +++++++ kalite/settings/__init__.py | 3 --- kalite/settings/base.py | 13 +++++++------ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/usermanual/userman_admin.rst b/docs/usermanual/userman_admin.rst index 15500c429b..6670e68d80 100644 --- a/docs/usermanual/userman_admin.rst +++ b/docs/usermanual/userman_admin.rst @@ -699,6 +699,13 @@ User restrictions Disables user sign ups. +..note:: KA Lite uses caching of web pages, if you change ``LOCKDOWN`` or + ``DISABLE_SELF_ADMIN``, you need to flush the cache. To do that, run + the following management command:: + + kalite manage cache clearweb + + Online Synchronization ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/kalite/settings/__init__.py b/kalite/settings/__init__.py index 718962ab6d..3d1e648eb4 100644 --- a/kalite/settings/__init__.py +++ b/kalite/settings/__init__.py @@ -74,9 +74,6 @@ def package_selected(package_name): if package_selected("UserRestricted"): LOG.info("UserRestricted package selected.") - - if CACHE_TIME != 0 and not hasattr(local_settings, KEY_PREFIX): - KEY_PREFIX += "|restricted" # this option changes templates DISABLE_SELF_ADMIN = True # hard-code facility app setting. diff --git a/kalite/settings/base.py b/kalite/settings/base.py index 254359155b..120008d231 100644 --- a/kalite/settings/base.py +++ b/kalite/settings/base.py @@ -447,11 +447,12 @@ } } +KEY_PREFIX = version.VERSION + # Cache is activated in every case, # EXCEPT: if CACHE_TIME=0 if CACHE_TIME != 0: # None can mean infinite caching to some functions # When we change versions, cache changes, too - KEY_PREFIX = ".".join(version.VERSION) # File-based cache install_location_hash = hashlib.sha1(".".join(version.VERSION)).hexdigest() @@ -460,10 +461,10 @@ CACHE_LOCATION = os.path.realpath(getattr(local_settings, "CACHE_LOCATION", os.path.join(tempfile.gettempdir(), cache_dir_name, install_location_hash))) + "/" CACHES["file_based_cache"] = { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': CACHE_LOCATION, # this is kind of OS-specific, so dangerous. - 'TIMEOUT': CACHE_TIME, # should be consistent + 'LOCATION': CACHE_LOCATION, # this is kind of OS-specific, so dangerous. + 'TIMEOUT': CACHE_TIME, # should be consistent 'OPTIONS': { - 'MAX_ENTRIES': getattr(local_settings, "CACHE_MAX_ENTRIES", 5*2000) #2000 entries=~10,000 files + 'MAX_ENTRIES': getattr(local_settings, "CACHE_MAX_ENTRIES", 5*2000) # 2000 entries=~10,000 files }, } @@ -471,9 +472,9 @@ CACHES["mem_cache"] = { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 'LOCATION': 'unique-snowflake', - 'TIMEOUT': CACHE_TIME, # should be consistent + 'TIMEOUT': CACHE_TIME, # should be consistent 'OPTIONS': { - 'MAX_ENTRIES': getattr(local_settings, "CACHE_MAX_ENTRIES", 5*2000) #2000 entries=~10,000 files + 'MAX_ENTRIES': getattr(local_settings, "CACHE_MAX_ENTRIES", 5*2000) # 2000 entries=~10,000 files }, } From c1615ad42562eb404970dae237aa25c1842b7ee7 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Mon, 20 Jul 2015 23:53:09 +0200 Subject: [PATCH 2/7] put membased cache in dev settings and switch to using the DEFAULT cache by DEFAULT, saving it to a persistent dir --- kalite/project/settings/dev.py | 9 +++++-- kalite/settings/base.py | 46 +++++++++++----------------------- 2 files changed, 21 insertions(+), 34 deletions(-) diff --git a/kalite/project/settings/dev.py b/kalite/project/settings/dev.py index c19a06cad9..fa5a6e4daf 100644 --- a/kalite/project/settings/dev.py +++ b/kalite/project/settings/dev.py @@ -66,8 +66,13 @@ 'debug_toolbar.panels.logging.LoggingPanel', 'debug_toolbar.panels.redirects.RedirectsPanel', # This belongs to DISABLE_PANELS by default ) + DEBUG_TOOLBAR_CONFIG = { 'ENABLE_STACKTRACES': True, } -# Debug toolbar must be set in conjunction with CACHE_TIME=0 -CACHE_TIME = 0 + +CACHES["default"] = { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'LOCATION': 'unique-snowflake', + 'TIMEOUT': 24 * 60 * 60 # = 24 hours +} diff --git a/kalite/settings/base.py b/kalite/settings/base.py index 120008d231..97df6f27e2 100644 --- a/kalite/settings/base.py +++ b/kalite/settings/base.py @@ -438,49 +438,31 @@ _100_years = 100 * 365 * 24 * 60 * 60 _max_cache_time = min(_100_years, sys.maxint - time.time() - _5_years) CACHE_TIME = getattr(local_settings, "CACHE_TIME", _max_cache_time) -CACHE_NAME = getattr(local_settings, "CACHE_NAME", None) # without a cache defined, None is fine # Sessions use the default cache, and we want a local memory cache for that. +CACHE_LOCATION = os.path.realpath(getattr( + local_settings, + "CACHE_LOCATION", + os.path.join( + USER_DATA_ROOT, + 'cache', + ) +)) + CACHES = { "default": { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - } -} - -KEY_PREFIX = version.VERSION - -# Cache is activated in every case, -# EXCEPT: if CACHE_TIME=0 -if CACHE_TIME != 0: # None can mean infinite caching to some functions - # When we change versions, cache changes, too - - # File-based cache - install_location_hash = hashlib.sha1(".".join(version.VERSION)).hexdigest() - username = getpass.getuser() or "unknown_user" - cache_dir_name = "kalite_web_cache_%s" % (username) - CACHE_LOCATION = os.path.realpath(getattr(local_settings, "CACHE_LOCATION", os.path.join(tempfile.gettempdir(), cache_dir_name, install_location_hash))) + "/" - CACHES["file_based_cache"] = { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', 'LOCATION': CACHE_LOCATION, # this is kind of OS-specific, so dangerous. 'TIMEOUT': CACHE_TIME, # should be consistent 'OPTIONS': { - 'MAX_ENTRIES': getattr(local_settings, "CACHE_MAX_ENTRIES", 5*2000) # 2000 entries=~10,000 files + 'MAX_ENTRIES': getattr(local_settings, "CACHE_MAX_ENTRIES", 5 * 2000) # 2000 entries=~10,000 files }, } +} - # Memory-based cache - CACHES["mem_cache"] = { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'unique-snowflake', - 'TIMEOUT': CACHE_TIME, # should be consistent - 'OPTIONS': { - 'MAX_ENTRIES': getattr(local_settings, "CACHE_MAX_ENTRIES", 5*2000) # 2000 entries=~10,000 files - }, - } - - # The chosen cache - CACHE_NAME = getattr(local_settings, "CACHE_NAME", "file_based_cache") - +# Prefix the cache with the version string so we don't experience problems with +# updates +KEY_PREFIX = version.VERSION # Separate session caching from file caching. SESSION_ENGINE = getattr( From 113b9bb779a25d26147eb459ef449d8760ce71c1 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Mon, 20 Jul 2015 23:58:02 +0200 Subject: [PATCH 3/7] unused imports --- python-packages/fle_utils/internet/webcache.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python-packages/fle_utils/internet/webcache.py b/python-packages/fle_utils/internet/webcache.py index 48c1f05360..3213675b5b 100644 --- a/python-packages/fle_utils/internet/webcache.py +++ b/python-packages/fle_utils/internet/webcache.py @@ -9,12 +9,10 @@ from django.core.cache.backends.filebased import FileBasedCache from django.core.cache.backends.locmem import LocMemCache from django.core.urlresolvers import reverse -from django.db.models.signals import post_save, pre_delete -from django.dispatch import receiver from django.http import HttpRequest from django.test.client import Client from django.utils import translation -from django.utils.cache import get_cache_key as django_get_cache_key, get_cache, _generate_cache_key +from django.utils.cache import get_cache_key as django_get_cache_key, get_cache from django.views.decorators.cache import cache_control from django.views.decorators.cache import cache_page from django.views.decorators.http import condition From 9e13a911082b7dd05129d473891e73fd2e3b6928 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Tue, 21 Jul 2015 00:00:03 +0200 Subject: [PATCH 4/7] do not use CACHE_NAME --- python-packages/fle_utils/internet/webcache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-packages/fle_utils/internet/webcache.py b/python-packages/fle_utils/internet/webcache.py index 3213675b5b..2ba1b761c3 100644 --- a/python-packages/fle_utils/internet/webcache.py +++ b/python-packages/fle_utils/internet/webcache.py @@ -64,7 +64,7 @@ def backend_cache_page(handler, cache_time=None, cache_name=None): cache_time = settings.CACHE_TIME if not cache_name: - cache_name = settings.CACHE_NAME + cache_name = "default" if caching_is_enabled(): @condition(last_modified_func=partial(calc_last_modified, cache_name=cache_name)) @@ -87,7 +87,7 @@ def caching_is_enabled(): return settings.CACHE_TIME != 0 def get_web_cache(): - return get_cache(settings.CACHE_NAME) if caching_is_enabled() else None + return get_cache('default') if caching_is_enabled() else None def get_cache_key(path=None, url_name=None, cache=None, failure_ok=False): From f92337984509732880ef1192e5ae4f62bf950769 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Tue, 21 Jul 2015 00:05:56 +0200 Subject: [PATCH 5/7] do not invalidate the web cache at startup --- kalite/distributed/management/commands/initialize_kalite.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/kalite/distributed/management/commands/initialize_kalite.py b/kalite/distributed/management/commands/initialize_kalite.py index da22a76790..de4c548650 100644 --- a/kalite/distributed/management/commands/initialize_kalite.py +++ b/kalite/distributed/management/commands/initialize_kalite.py @@ -38,9 +38,6 @@ def setup_server_if_needed(self): def reinitialize_server(self): """Reset the server state.""" - logging.info("Invalidating the web cache.") - from fle_utils.internet.webcache import invalidate_web_cache - invalidate_web_cache() # Next, call videoscan. logging.info("Running videoscan.") From c2794b2a0c94ba92ced748ebf89fbd452462d23f Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Tue, 21 Jul 2015 00:16:43 +0200 Subject: [PATCH 6/7] unused import --- kalite/distributed/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/kalite/distributed/views.py b/kalite/distributed/views.py index 58fa7ab3c8..ad6c4b7227 100755 --- a/kalite/distributed/views.py +++ b/kalite/distributed/views.py @@ -24,8 +24,6 @@ from fle_utils.internet.classes import JsonResponseMessageError from fle_utils.internet.functions import get_ip_addresses, set_query_params -from fle_utils.internet.webcache import backend_cache_page -from fle_utils.django_utils.paginate import paginate_data from kalite import topic_tools from kalite.shared.decorators.auth import require_admin from securesync.api_client import BaseClient From af34eee9ef896f8e15643d56ef5e2dede224d8be Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Tue, 21 Jul 2015 00:34:14 +0200 Subject: [PATCH 7/7] do not use disk-based cache for what's already in memory --- kalite/caching/__init__.py | 2 +- kalite/main/api_views.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/kalite/caching/__init__.py b/kalite/caching/__init__.py index 9f2c71f94b..bb0c8f5467 100644 --- a/kalite/caching/__init__.py +++ b/kalite/caching/__init__.py @@ -24,7 +24,7 @@ from django.core.urlresolvers import reverse from django.test.client import Client -from fle_utils.internet.webcache import * +from fle_utils.internet.webcache import get_web_cache, has_cache_key, expire_page, caching_is_enabled, invalidate_web_cache from kalite import i18n, topic_tools from kalite.topic_tools.settings import DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP diff --git a/kalite/main/api_views.py b/kalite/main/api_views.py index f240aa6a39..0fa29a58c1 100755 --- a/kalite/main/api_views.py +++ b/kalite/main/api_views.py @@ -9,14 +9,12 @@ from fle_utils.internet.decorators import api_handle_error_with_json from fle_utils.internet.classes import JsonResponse, JsonResponseMessageError -from fle_utils.internet.webcache import backend_cache_page from kalite.topic_tools import get_topic_tree from kalite.topic_tools.content_recommendation import get_resume_recommendations, get_next_recommendations, get_explore_recommendations from kalite.facility.models import FacilityUser @api_handle_error_with_json -@backend_cache_page def topic_tree(request, channel): parent = request.GET.get("parent") return JsonResponse(get_topic_tree(channel=channel, language=request.language, parent=parent))